From d4f7ccaff4a9a91e19a6494d93294b4d016af3fb Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Fri, 9 Oct 2026 13:55:18 +0800 Subject: [PATCH 1/2] passthrough: test the readdir and readdirplus contracts The existing coverage stops at buffer parsing (readdir_cookie_tests), the no_opendir re-seek regression (generic/676), and shallow error paths of test_dir_operations. The persistent-opendir flow that the kernel actually drives -- opendir handle, cookie resume, batch splitting, the readdirplus attribute and ownership contracts -- had no coverage at all. Add a readdir_tests module exercising that flow against a real backing directory through a persistent handle: - enumeration: a stream that does not fit one reply batch is drained completely and exactly once by resuming from the cookie of the last delivered entry, with strictly increasing non-zero cookies and a stable order across repeated enumerations of the same handle; - readdirplus contract: the dirent ino equals the accompanying Entry's attr.st_ino, the Entry matches a plain lookup of the name (inode, mode, size), and delivered entries stay mapped with exactly the kernel-owned reference until forget() drops them; - ownership: plain readdir is reference-neutral (transient per-entry lookups are forgotten again), and an entry rejected by a full reply buffer (add_entry -> 0) has its readdirplus reference released immediately instead of leaking; - error contract: an error before anything was stored propagates (exercised on a resumed stream, since a fresh one begins with the internally filtered "." and ".."), an error after a delivered entry returns Ok(()) with the partial delivery; - cookies: resuming from a mid-stream cookie repositions via lseek, a cookie beyond i64::MAX terminates at EOF through the linear scan fallback, and releasedir drops the cached cookie together with the handle (the cached cookie is a one-shot token consumed by the matching resume, which the drain helper's final EOF probe does); - plus the d_type passthrough of REG/DIR/LNK entries, the empty directory, size == 0, and the no_readdir kill switch. The setup uses a plain path-based config without inode file handles, so the suite runs unprivileged like the rest of the crate's non-privileged tests. Signed-off-by: Jiang Liu --- .../fuse-backend-passthrough/src/sync_io.rs | 540 ++++++++++++++++++ 1 file changed, 540 insertions(+) diff --git a/crates/fuse-backend-passthrough/src/sync_io.rs b/crates/fuse-backend-passthrough/src/sync_io.rs index a8bb1759..236ddf85 100644 --- a/crates/fuse-backend-passthrough/src/sync_io.rs +++ b/crates/fuse-backend-passthrough/src/sync_io.rs @@ -2546,3 +2546,543 @@ mod readdir_cookie_tests { assert_eq!(buf, corrupted); } } + +// End-to-end readdir/readdirplus coverage against a real backing directory: +// enumeration semantics (batching, cookies, EOF), the readdirplus +// attribute contract, the lookup-refcount ownership of the two handlers, +// the partial-delivery error contract, and the cookie cache lifecycle. +#[cfg(test)] +mod readdir_tests { + use super::*; + use fuse_backend_core::abi::fuse_abi::ROOT_ID; + use vmm_sys_util::tempdir::TempDir; + + /// A passthrough fs over a fresh temporary directory, without inode + /// file handles: the readdir paths under test are purely path-based, + /// and `open_by_handle_at()` would only add an unprivileged EPERM + /// failure mode on hosts where file handles are unavailable. + fn prepare_fs() -> (PassthroughFs<()>, TempDir) { + let source = TempDir::new().expect("Cannot create temporary directory."); + let fs_cfg = Config { + writeback: true, + do_import: true, + root_dir: source + .as_path() + .to_str() + .expect("source path to string") + .to_string(), + ..Default::default() + }; + let fs = PassthroughFs::<()>::new(fs_cfg).unwrap(); + // do_import: true, so init() imports the root inode; the ZERO_MESSAGE + // options are not negotiated because cfg.no_open/no_opendir are false, + // keeping persistent opendir() handles available to the tests. + fs.init(FsOptions::all()).unwrap(); + (fs, source) + } + + fn prepare_context() -> Context { + Context { + uid: unsafe { libc::getuid() }, + gid: unsafe { libc::getgid() }, + pid: unsafe { libc::getpid() }, + ..Default::default() + } + } + + /// Create `count` regular files named `f%03d` directly in the backing + /// directory, bypassing the fs interfaces: the readdir handlers must + /// pick them up through do_lookup() without any prior interaction, so + /// the tests can observe exactly the references they take themselves. + fn create_backing_files(source: &TempDir, count: usize) { + for i in 0..count { + std::fs::File::create(source.as_path().join(format!("f{:03}", i))) + .expect("create file"); + } + } + + /// One delivered readdir entry. + struct Dirent { + offset: u64, + ino: u64, + type_: u32, + name: Vec, + } + + /// One delivered readdirplus entry: the dirent fields plus the pieces of + /// the lookup `Entry` that the readdirplus contract ties together. + struct DirentPlus { + dirent: Dirent, + inode: u64, + st_ino: u64, + st_mode: u32, + st_size: i64, + } + + /// Drain a whole directory stream through `readdir`, resuming from the + /// cookie of the last delivered entry until EOF, exactly like the fuse + /// kernel client does. The batch count is bounded so that a cookie bug + /// fails the test instead of looping forever. + fn readdir_all( + fs: &PassthroughFs<()>, + ctx: &Context, + handle: Handle, + size: u32, + ) -> Vec { + let mut out: Vec = Vec::new(); + let mut offset = 0_u64; + for _ in 0..1024 { + let mut batch = Vec::new(); + fs.readdir(ctx, ROOT_ID, handle, size, offset, &mut |e| { + batch.push(Dirent { + offset: e.offset, + ino: e.ino, + type_: e.type_, + name: e.name.to_vec(), + }); + Ok(1) + }) + .unwrap(); + if batch.is_empty() { + break; + } + offset = batch.last().unwrap().offset; + out.extend(batch); + } + out + } + + /// Drain a whole directory stream through `readdirplus`, same resume + /// discipline as `readdir_all()`. + fn readdirplus_all( + fs: &PassthroughFs<()>, + ctx: &Context, + handle: Handle, + size: u32, + ) -> Vec { + let mut out: Vec = Vec::new(); + let mut offset = 0_u64; + for _ in 0..1024 { + let mut batch = Vec::new(); + fs.readdirplus(ctx, ROOT_ID, handle, size, offset, &mut |e, entry| { + batch.push(DirentPlus { + dirent: Dirent { + offset: e.offset, + ino: e.ino, + type_: e.type_, + name: e.name.to_vec(), + }, + inode: entry.inode, + st_ino: entry.attr.st_ino, + st_mode: entry.attr.st_mode, + st_size: entry.attr.st_size, + }); + Ok(1) + }) + .unwrap(); + if batch.is_empty() { + break; + } + offset = batch.last().unwrap().dirent.offset; + out.extend(batch); + } + out + } + + /// Current refcount of `ino` in the inode map, or None when the inode is + /// not mapped at all. + fn refcount(fs: &PassthroughFs<()>, ino: Inode) -> Option { + fs.inode_map + .inodes + .read() + .unwrap() + .get(&ino) + .map(|d| d.refcount.load(std::sync::atomic::Ordering::Acquire)) + } + + fn open_root(fs: &PassthroughFs<()>, ctx: &Context) -> Handle { + let (handle, _) = fs + .opendir(ctx, ROOT_ID, libc::O_RDONLY as u32) + .expect("opendir"); + handle.expect("opendir handle") + } + + // A directory stream that does not fit into a single reply batch must + // be delivered completely and exactly once: every resume goes through + // the cached-cookie fast path of do_readdir(), so this pins both the + // batch splitting and the cookie bookkeeping of the persistent handle. + #[test] + fn readdir_enumerates_all_entries_in_batches() { + let (fs, source) = prepare_fs(); + let ctx = prepare_context(); + create_backing_files(&source, 64); + + let handle = open_root(&fs, &ctx); + // A size of 128 bytes holds only a handful of dirents, forcing the + // stream into ~20 reply batches. + let entries = readdir_all(&fs, &ctx, handle, 128); + + assert_eq!(entries.len(), 64); + let mut names: Vec<&Vec> = entries.iter().map(|e| &e.name).collect(); + names.sort(); + names.dedup(); + assert_eq!(names.len(), 64, "duplicate or lost entries"); + assert!(entries.iter().all(|e| e.offset != 0)); + assert!( + entries.windows(2).all(|w| w[0].offset < w[1].offset), + "cookies must strictly increase across the stream" + ); + assert!(entries.iter().all(|e| e.ino != 0)); + + // A second full enumeration of the same handle must be identical: + // resuming from 0 re-seeks the stream, and the entry order of a + // getdents64 stream is stable within one fd. + let again = readdir_all(&fs, &ctx, handle, 128); + assert_eq!( + entries + .iter() + .map(|e| (e.offset, e.name.clone())) + .collect::>(), + again + .iter() + .map(|e| (e.offset, e.name.clone())) + .collect::>() + ); + + fs.releasedir(&ctx, ROOT_ID, 0, handle).unwrap(); + } + + // The readdirplus contract: the dirent inode must equal the st_ino of + // the accompanying Entry, the Entry must describe the same file a + // lookup() of the name returns, and the attributes must be real + // (correct type and size). Delivered entries must also stay mapped + // until the client forgets them. + #[test] + fn readdirplus_attrs_match_lookup() { + let (fs, source) = prepare_fs(); + let ctx = prepare_context(); + let count = 32; + for i in 0..count { + let size = (i as u64) * 17; + std::fs::write( + source.as_path().join(format!("f{:03}", i)), + vec![0u8; size as usize], + ) + .expect("write file"); + } + + let handle = open_root(&fs, &ctx); + let entries = readdirplus_all(&fs, &ctx, handle, 8192); + assert_eq!(entries.len(), count); + + for e in &entries { + assert_eq!(e.dirent.ino, e.st_ino, "dirent ino must equal attr.st_ino"); + assert_eq!(e.dirent.type_, libc::DT_REG as u32); + assert_eq!(e.st_mode & libc::S_IFMT, libc::S_IFREG); + + let name = CString::new(e.dirent.name.clone()).unwrap(); + let looked_up = fs.lookup(&ctx, ROOT_ID, &name).unwrap(); + assert_eq!(e.inode, looked_up.inode, "entry inode must match lookup"); + assert_eq!(e.st_ino, looked_up.attr.st_ino); + assert_eq!(e.st_mode, looked_up.attr.st_mode); + // Drop the reference the verification lookup took again, so the + // refcount checks below observe only the readdirplus ones. + fs.forget(&ctx, looked_up.inode, 1); + + let idx: usize = std::str::from_utf8(&e.dirent.name) + .unwrap() + .trim_start_matches('f') + .parse() + .unwrap(); + assert_eq!(e.st_size, (idx as i64) * 17); + } + + // readdirplus delivers entries with the lookup reference retained + // (the kernel owns it until a FORGET), so every child is mapped + // with exactly one reference after enumeration. + for e in &entries { + assert_eq!(refcount(&fs, e.inode), Some(1)); + } + // Forgetting the delivered references must drop the mappings. + for e in &entries { + fs.forget(&ctx, e.inode, 1); + assert_eq!(refcount(&fs, e.inode), None); + } + + fs.releasedir(&ctx, ROOT_ID, 0, handle).unwrap(); + } + + // Plain readdir must be reference-neutral: it looks each entry up to + // learn its inode and immediately forgets that reference again. Files + // never seen before stay unmapped, and a file the caller holds a + // reference to keeps exactly that reference. + #[test] + fn readdir_releases_lookup_references() { + let (fs, source) = prepare_fs(); + let ctx = prepare_context(); + create_backing_files(&source, 8); + + // One file with a caller-held reference: its refcount must survive + // any number of enumerations unchanged. + let held = CString::new("f003").unwrap(); + let held_entry = fs.lookup(&ctx, ROOT_ID, &held).unwrap(); + assert_eq!(refcount(&fs, held_entry.inode), Some(1)); + + let handle = open_root(&fs, &ctx); + for round in 0..2 { + let entries = readdir_all(&fs, &ctx, handle, 4096); + assert_eq!(entries.len(), 8, "round {}", round); + + for e in &entries { + if e.ino == held_entry.inode { + // The pre-existing reference is untouched. + assert_eq!(refcount(&fs, e.ino), Some(1)); + } else { + // The reference taken by the handler was forgotten. + assert_eq!(refcount(&fs, e.ino), None, "round {}", round); + } + } + } + + fs.releasedir(&ctx, ROOT_ID, 0, handle).unwrap(); + assert_eq!(refcount(&fs, held_entry.inode), Some(1)); + } + + // An entry that does not fit into the reply buffer (add_entry returning + // 0) is not delivered, so readdirplus must release its lookup reference + // right away instead of leaking it -- and must not report an error. + #[test] + fn readdirplus_buffer_full_forgets_undelivered() { + let (fs, source) = prepare_fs(); + let ctx = prepare_context(); + create_backing_files(&source, 4); + + let handle = open_root(&fs, &ctx); + let mut seen = 0; + let mut first_ino = 0; + fs.readdirplus(&ctx, ROOT_ID, handle, 8192, 0, &mut |_e, entry| { + seen += 1; + first_ino = entry.inode; + Ok(0) + }) + .unwrap(); + assert_eq!(seen, 1); + + // The undelivered entry was looked up and forgotten again. + assert_eq!(refcount(&fs, first_ino), None); + assert_ne!(first_ino, 0, "an inode must have been observed"); + + fs.releasedir(&ctx, ROOT_ID, 0, handle).unwrap(); + } + + // Error contract of the reply loop: an error before any entry was + // stored propagates to the caller; an error after at least one entry + // was stored returns Ok(()) with the partial delivery, because the + // entries already handed out cannot be taken back. A fresh stream + // always begins with "." and "..", which are filtered without the + // callback, so the propagate case is exercised on a resumed stream + // whose first record is a real entry. + #[test] + fn readdir_error_contract() { + let (fs, source) = prepare_fs(); + let ctx = prepare_context(); + create_backing_files(&source, 4); + + let handle = open_root(&fs, &ctx); + let entries = readdir_all(&fs, &ctx, handle, 8192); + assert_eq!(entries.len(), 4); + + let eio = || io::Error::from_raw_os_error(libc::EIO); + let err = fs + .readdir(&ctx, ROOT_ID, handle, 8192, entries[0].offset, &mut |_| { + Err(eio()) + }) + .unwrap_err(); + assert_eq!( + err.raw_os_error(), + Some(libc::EIO), + "the callback's error must propagate on the first entry" + ); + + let mut delivered = 0; + fs.readdir(&ctx, ROOT_ID, handle, 8192, 0, &mut |_| { + delivered += 1; + if delivered == 1 { + Ok(1) + } else { + Err(eio()) + } + }) + .unwrap(); + assert_eq!( + delivered, 2, + "callbacks: one entry delivered, then the error" + ); + + fs.releasedir(&ctx, ROOT_ID, 0, handle).unwrap(); + } + + // An empty directory enumerates to EOF immediately, and size == 0 is + // answered without touching the directory at all. + #[test] + fn readdir_empty_directory() { + let (fs, _source) = prepare_fs(); + let ctx = prepare_context(); + + let handle = open_root(&fs, &ctx); + assert!(readdir_all(&fs, &ctx, handle, 4096).is_empty()); + assert!(readdirplus_all(&fs, &ctx, handle, 4096).is_empty()); + + let mut entries = 0; + fs.readdir(&ctx, ROOT_ID, handle, 0, 0, &mut |_| { + entries += 1; + Ok(1) + }) + .unwrap(); + assert_eq!(entries, 0, "size == 0 must not enumerate anything"); + + fs.releasedir(&ctx, ROOT_ID, 0, handle).unwrap(); + } + + // Resuming from a cookie that is not the one cached for the handle (the + // kernel re-issuing an earlier offset) must reposition the stream via + // lseek and continue with the following entry: both when the cache is + // empty and when it holds a stale cookie that must be discarded. A + // cookie that lseek cannot represent (> i64::MAX, e.g. an NFSv4 cookie) + // takes the linear scan fallback and terminates at EOF without looping. + #[test] + fn readdir_resume_from_mid_stream_cookie() { + let (fs, source) = prepare_fs(); + let ctx = prepare_context(); + create_backing_files(&source, 16); + + let handle = open_root(&fs, &ctx); + let entries = readdir_all(&fs, &ctx, handle, 8192); + assert_eq!(entries.len(), 16); + + // The drain consumed the one-shot cached cookie with its final EOF + // probe, so the cache is empty: resuming from a mid-stream cookie is + // a miss against the empty cache and must go through lseek. + let k = 5; + let mut first = None; + fs.readdir(&ctx, ROOT_ID, handle, 8192, entries[k].offset, &mut |e| { + if first.is_none() { + first = Some(e.name.to_vec()); + } + Ok(1) + }) + .unwrap(); + assert_eq!(first, Some(entries[k + 1].name.clone())); + + // A partial reply leaves the cookie of its last entry cached while + // the stream is mid-way. Resuming from an earlier cookie mismatches + // that token: it must be discarded rather than trusted as the fd + // position, and the stream repositioned via lseek, continuing with + // the entry after the requested one. + let mut n = 0; + fs.readdir(&ctx, ROOT_ID, handle, 128, 0, &mut |_| { + n += 1; + Ok(1) + }) + .unwrap(); + assert!(n > 0 && n < 16, "a 128-byte reply must be a partial batch"); + let mut mismatch_first = None; + fs.readdir(&ctx, ROOT_ID, handle, 8192, entries[0].offset, &mut |e| { + if mismatch_first.is_none() { + mismatch_first = Some(e.name.to_vec()); + } + Ok(1) + }) + .unwrap(); + assert_eq!(mismatch_first, Some(entries[1].name.clone())); + + // A cookie beyond i64::MAX cannot be seeked to; the scan fallback + // walks the directory without finding it and reports EOF. + let mut entries_after_huge = 0; + fs.readdir(&ctx, ROOT_ID, handle, 8192, u64::MAX - 1, &mut |_| { + entries_after_huge += 1; + Ok(1) + }) + .unwrap(); + assert_eq!(entries_after_huge, 0, "unknown huge cookie must end at EOF"); + + fs.releasedir(&ctx, ROOT_ID, 0, handle).unwrap(); + } + + // The d_type bits of the getdents records must reach the client + // unchanged for the entry kinds a directory can hold. + #[test] + fn readdir_reports_entry_types() { + let (fs, source) = prepare_fs(); + let ctx = prepare_context(); + create_backing_files(&source, 1); + std::fs::create_dir(source.as_path().join("subdir")).expect("create dir"); + std::os::unix::fs::symlink("f000", source.as_path().join("link")).expect("create symlink"); + + let handle = open_root(&fs, &ctx); + let entries = readdir_all(&fs, &ctx, handle, 8192); + + let by_name = |name: &str| { + entries + .iter() + .find(|e| e.name == name.as_bytes()) + .unwrap_or_else(|| panic!("entry {} missing", name)) + }; + assert_eq!(by_name("f000").type_, libc::DT_REG as u32); + assert_eq!(by_name("subdir").type_, libc::DT_DIR as u32); + assert_eq!(by_name("link").type_, libc::DT_LNK as u32); + + fs.releasedir(&ctx, ROOT_ID, 0, handle).unwrap(); + } + + // In no_readdir mode both handlers report success without enumerating + // anything, letting the client fall back to lookup-based iteration. + #[test] + fn readdir_disabled_returns_empty() { + let (fs, source) = prepare_fs(); + let ctx = prepare_context(); + create_backing_files(&source, 4); + fs.no_readdir.store(true, Ordering::Relaxed); + + let handle = open_root(&fs, &ctx); + assert!(readdir_all(&fs, &ctx, handle, 4096).is_empty()); + assert!(readdirplus_all(&fs, &ctx, handle, 4096).is_empty()); + + fs.releasedir(&ctx, ROOT_ID, 0, handle).unwrap(); + } + + // releasedir must drop the handle together with the cookie cached for + // its directory stream, and the released handle must stop working. + #[test] + fn releasedir_clears_the_cookie_cache() { + let (fs, source) = prepare_fs(); + let ctx = prepare_context(); + create_backing_files(&source, 4); + + let handle = open_root(&fs, &ctx); + // One served batch leaves the cookie of its last entry in the cache. + // Draining the whole stream would consume the token again: the + // cached cookie is a one-shot fast path for the next resume, and the + // final EOF probe of a full enumeration matches it. + let mut seen = 0; + fs.readdir(&ctx, ROOT_ID, handle, 4096, 0, &mut |_| { + seen += 1; + Ok(1) + }) + .unwrap(); + assert!(seen > 0); + assert!( + !fs.handle_map.cookies.lock().unwrap().is_empty(), + "a served readdir must have cached a cookie" + ); + + fs.releasedir(&ctx, ROOT_ID, 0, handle).unwrap(); + assert!( + fs.handle_map.cookies.lock().unwrap().is_empty(), + "releasedir must drop the cached cookie" + ); + assert!(fs + .readdir(&ctx, ROOT_ID, handle, 4096, 0, &mut |_| Ok(1)) + .is_err()); + } +} From cc35776e71f9ef84019e55e9c2b954d84e348d47 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Fri, 9 Oct 2026 13:55:35 +0800 Subject: [PATCH 2/2] benchmark: add a readdir workload The fio comparison covers data IO and create/delete metadata, but not directory enumeration -- the READDIR/READDIRPLUS handlers, whose sync vs async cost profile is exactly the open question of the offload discussion. fio has no readdir engine, so drive it with a helper instead: bench_readdir.py walks the mountpoint with os.scandir() (getdents64) over and over, first for a 2 second ramp whose counts are discarded, then for ${RUNTIME} seconds measured by the wall clock that spans the whole window. It writes fio-shaped JSON (entries/s as iops on the read side) so bench_compare.py and the CI collect/compare steps treat it like any other workload without any special casing. Run it between filecreate and filedelete, when the directory holds ${NRFILES} entries: the workload is self-contained in the create/ delete pair and sees a realistic directory size. Enumeration is one strictly sequential request chain per stream -- each getdents64 resumes from the cookie of the previous reply -- so the rate is latency-bound and does not scale with THREADS. Classify readdir with the metadata workloads for the looser regression threshold: it is a per-entry metadata operation like create/delete, and its byte metrics are only approximated reply traffic anyway. On an unprivileged host the run reports ~210k entries/s with sync and async at parity (a 5 second smoke run on a 20k-entry directory). Signed-off-by: Jiang Liu --- tests/benchmark/README.md | 14 +++- tests/scripts/bench_compare.py | 7 +- tests/scripts/bench_readdir.py | 119 ++++++++++++++++++++++++++++++ tests/scripts/bench_sync_async.sh | 36 +++++++-- 4 files changed, 167 insertions(+), 9 deletions(-) create mode 100644 tests/scripts/bench_readdir.py diff --git a/tests/benchmark/README.md b/tests/benchmark/README.md index 4c152833..f0e65c48 100644 --- a/tests/benchmark/README.md +++ b/tests/benchmark/README.md @@ -37,9 +37,16 @@ otherwise) — and runs identical fio workloads: - sequential read/write with 1MB requests - random read/write with 4KB requests - metadata operations (file create/delete) +- directory enumeration (readdir): fio has no readdir engine, so the + `bench_readdir.py` helper walks the mountpoint with `os.scandir()` + (getdents64) between the filecreate and filedelete workloads, when the + directory holds `NRFILES` entries, and reports entries/s as `iops` in + the same JSON job shape. Enumeration is one strictly sequential request + chain per stream (each getdents64 resumes from the cookie of the previous + reply), so the rate is latency-bound and does not scale with `THREADS`. ```sh -sudo tests/scripts/bench_sync_async.sh # needs fio + fuse mount rights +sudo tests/scripts/bench_sync_async.sh # needs fio + jq + python3 + fuse mount rights THREADS=8 RUNTIME=60 sudo -E tests/scripts/bench_sync_async.sh ``` @@ -114,8 +121,9 @@ value. Several measures keep the measurements stable: the data workloads use a 2 second fio ramp phase to exclude cold-start effects, a `sync` flushes -writeback between workloads, and the metadata workloads create/delete -50000 files each so that they run long enough to measure reliably. +writeback between workloads, the metadata workloads create/delete +50000 files each so that they run long enough to measure reliably, and +the readdir helper discards its own 2 second ramp phase before measuring. Because the metadata workloads still fluctuate more than the data ones even on bare metal, they are flagged with a looser threshold (25% versus 10%). diff --git a/tests/scripts/bench_compare.py b/tests/scripts/bench_compare.py index 28710623..5e36e8ac 100755 --- a/tests/scripts/bench_compare.py +++ b/tests/scripts/bench_compare.py @@ -21,6 +21,8 @@ where the metrics are taken from the read side of the fio job if it transferred data, otherwise from the write side. `runtime_ms` is only meaningful for the fixed-amount workloads (filecreate/filedelete). +The readdir workload is not driven by fio but by bench_readdir.py, which +reports entries/s as `iops` in the same JSON job shape. `compare` prints a markdown table and always exits with 0: shared CI runners are noisy, so the report is advisory and not a merge gate. @@ -37,7 +39,10 @@ # The metadata workloads fluctuate much more than the data workloads even # on bare metal (up to ~25% between runs of identical code), so flag them # with a looser threshold to keep the false-positive rate comparable. -METADATA_WORKLOADS = ("filecreate", "filedelete") +# readdir is a per-entry metadata operation like create/delete (and its +# byte metrics are only approximated reply traffic), so it shares the +# conservative class until bare-metal variance data justifies tightening. +METADATA_WORKLOADS = ("filecreate", "filedelete", "readdir") METADATA_THRESHOLD = 25.0 diff --git a/tests/scripts/bench_readdir.py b/tests/scripts/bench_readdir.py new file mode 100644 index 00000000..1ae00bf9 --- /dev/null +++ b/tests/scripts/bench_readdir.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# Copyright 2026 Alibaba Cloud. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 +"""Readdir workload: repeated directory enumeration for a fixed time. + +fio has no directory-enumeration engine, so bench_sync_async.sh calls this +helper instead: it walks the mountpoint with os.scandir() (getdents64) over +and over, first for a 2 second ramp whose counts are discarded, then for the +requested measurement time. Every enumeration is streamed to the fuse +daemon (the kernel does not serve fuse readdir from its dentry cache), so +the reported rate measures the READDIR/READDIRPLUS handlers of the mounted +file system, with READDIRPLUS negotiated by the benchmark daemon. + +The output file uses the fio JSON job shape so that bench_compare.py and +the CI tables treat it like any other fio workload: + iops directory entries delivered per second + bw approximated reply traffic, KiB/s + io_bytes approximated reply traffic, bytes + runtime measured window in ms +The byte figures approximate the READDIRPLUS reply size: a fuse dirent +plus its entry attributes is roughly 152 bytes plus the name, and the +kernel may fall back to plain READDIR records (about a third of that) +once its dentry cache is warm, so treat them as an upper bound; the +entries-per-second rate is the meaningful metric. + +Usage: bench_readdir.py +""" +import json +import os +import sys +import time + +# Ramp phase in seconds, discarded from the measurement like fio ramp_time. +RAMP_SECONDS = 2.0 +# sizeof(fuse_entry_out) + sizeof(fuse_dirent) on 64-bit hosts: the size of +# one READDIRPLUS reply entry before the (8-byte padded) name. +DIRENTPLUS_BYTES = 152 + + +def enumerate_once(path): + """One full getdents64 walk of `path`; returns (entries, name_bytes).""" + entries = 0 + name_bytes = 0 + with os.scandir(path) as it: + for entry in it: + entries += 1 + name_bytes += len(entry.name) + return entries, name_bytes + + +def main(): + if len(sys.argv) != 4: + print("usage: bench_readdir.py " + "") + sys.exit(2) + path, out_path = sys.argv[1], sys.argv[3] + try: + runtime = float(sys.argv[2]) + except ValueError: + print("runtime must be a number of seconds") + sys.exit(2) + if runtime <= 0: + print("runtime must be positive") + sys.exit(2) + + # Fail loudly instead of reporting a hot zero-entries spin loop: the + # workload is meant to run after filecreate, when the directory holds + # ${NRFILES} entries. + if enumerate_once(path)[0] == 0: + print("no entries to enumerate under %s" % path) + sys.exit(1) + + # Warm the daemon and kernel caches so the measurement window covers + # the steady state, like the fio ramp_time of the data workloads. + ramp_deadline = time.monotonic() + RAMP_SECONDS + while time.monotonic() < ramp_deadline: + enumerate_once(path) + + entries = 0 + reply_bytes = 0 + # The deadline is only checked between walks, so the window may + # overshoot by at most one enumeration; dividing by the actually + # elapsed time keeps the reported rate honest. + start = time.monotonic() + deadline = start + runtime + while time.monotonic() < deadline: + count, name_bytes = enumerate_once(path) + entries += count + reply_bytes += count * DIRENTPLUS_BYTES + name_bytes + elapsed = time.monotonic() - start + + # Same shape as one fio job: the read side carries the work, the write + # side stays zero, and collect() of bench_compare.py picks the active + # side by exactly that difference. bw is an integer like fio's so the + # side-by-side summary of the script stays aligned; iops keeps full + # precision like fio's. + job = { + "jobname": "readdir", + "read": { + "io_bytes": reply_bytes, + "bw": round(reply_bytes / elapsed / 1024.0), + "iops": entries / elapsed, + "runtime": int(elapsed * 1000), + }, + "write": { + "io_bytes": 0, + "bw": 0.0, + "iops": 0.0, + "runtime": 0, + }, + } + with open(out_path, "w", encoding="utf-8") as f: + json.dump({"jobs": [job]}, f, indent=2) + f.write("\n") + + +if __name__ == "__main__": + main() diff --git a/tests/scripts/bench_sync_async.sh b/tests/scripts/bench_sync_async.sh index e020e787..202c960b 100755 --- a/tests/scripts/bench_sync_async.sh +++ b/tests/scripts/bench_sync_async.sh @@ -13,7 +13,7 @@ # ($RESULTS_DIR/-.json) under $RESULTS_DIR for further # analysis, see tests/scripts/bench_compare.py. # -# Requirements: Linux, fio, jq, permission to mount fuse (root or +# Requirements: Linux, fio, jq, python3, permission to mount fuse (root or # fusermount). The uring mode additionally requires kernel 6.14+ with # FUSE-over-io_uring enabled (the fuse module parameter enable_uring, # which the script tries to turn on when it is writable); kernels that @@ -55,10 +55,11 @@ SRC_DIR="${RESULTS_DIR}/source" MNT_DIR="${RESULTS_DIR}/mount" mkdir -p "${SRC_DIR}" "${MNT_DIR}" -WORKLOADS="seqwrite seqread randwrite-4k randread-4k filecreate filedelete" +WORKLOADS="seqwrite seqread randwrite-4k randread-4k filecreate readdir filedelete" command -v fio >/dev/null 2>&1 || { echo "error: fio is required" >&2; exit 1; } command -v jq >/dev/null 2>&1 || { echo "error: jq is required" >&2; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo "error: python3 is required" >&2; exit 1; } # kernel_at_least MAJOR MINOR: true if the running kernel is at least # MAJOR.MINOR. Used to gate the uring mode, which requires the kernel @@ -187,6 +188,28 @@ run_fixed_workload() { end' "${RESULTS_DIR}/${mode}-${name}.json" || true } +# run_readdir_workload +# Directory enumeration: fio has no readdir engine, so bench_readdir.py +# walks the mountpoint with getdents64 for ${RUNTIME} seconds and writes +# fio-shaped JSON for bench_compare.py. Runs right after filecreate, when +# the directory holds ${NRFILES} entries, and before filedelete removes +# them again. One enumeration is a strictly sequential request chain (each +# getdents64 resumes from the cookie of the previous reply), so the rate +# is latency-bound and does not scale with ${THREADS}. +run_readdir_workload() { + local mode=$1 + local name=readdir + echo "--- [${mode}] ${name}: enumerating ${MNT_DIR} for ${RUNTIME}s" + sync + python3 "${SCRIPT_DIR}/bench_readdir.py" "${MNT_DIR}" "${RUNTIME}" \ + "${RESULTS_DIR}/${mode}-${name}.json" || { + echo "error: readdir workload failed" >&2 + exit 1 + } + jq -r '.jobs[0] | " read: iops=\(.read.iops | round) entries/s"' \ + "${RESULTS_DIR}/${mode}-${name}.json" || true +} + # shellcheck disable=SC2086 for mode in ${MODES}; do mode_args="" @@ -234,11 +257,14 @@ for mode in ${MODES}; do run_workload "${mode}" randread-4k --rw=randread --bs=4k --size=64M \ --numjobs="${THREADS}" --ioengine=psync --ramp_time=2 - # Metadata operations: create and delete lots of small files. These run - # for a fixed number of files instead of ${RUNTIME} seconds, see - # run_fixed_workload(). + # Metadata operations: create lots of small files, enumerate the + # directory they live in, then delete the files again. Create and + # delete run for a fixed number of files instead of ${RUNTIME} seconds, + # see run_fixed_workload(); the enumeration runs for ${RUNTIME} seconds, + # see run_readdir_workload(). run_fixed_workload "${mode}" filecreate --ioengine=filecreate \ --nrfiles="${NRFILES}" --numjobs=1 --filesize=4K + run_readdir_workload "${mode}" run_fixed_workload "${mode}" filedelete --ioengine=filedelete \ --nrfiles="${NRFILES}" --numjobs=1 --filesize=4K