From ede9cce4b537b0e2bba3cf6d72eefa68af3b5283 Mon Sep 17 00:00:00 2001 From: Codex GPT-5 Date: Sat, 25 Jul 2026 07:44:21 +0200 Subject: [PATCH 1/2] feat!: accelerate gix-dir walks with UNTR caches. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replay compatible untracked-cache directory records after validating worktree stats (including nanoseconds when serialized), per-directory and global excludes, the complete location-and-system cache identity, flags, and traversal options. Repository-inspecting walks and invalid or unsupported cache state fall back to the existing filesystem walk for the affected subtree so inherited ignore changes cannot leave descendant caches stale. Accept Git’s raw and newline-adjusted tracked .gitignore hashes. Add Criterion scenarios built with gix-testtools and prime every generated dirwalk fixture with Git. The benchmark medians improve as follows: - clean flat: 5.083 ms -> 116.58 us (-97.7%) - clean wide: 5.969 ms -> 277.07 us (-95.4%) - untracked wide: 5.804 ms -> 456.90 us (-92.1%) Git dir.c and t/t7063-status-untracked-cache.sh are the behavioral reference. Regression coverage verifies identical entries with fewer read_dir calls, invalidates descendants after inherited ignore changes, and exercises UNTR eligibility across the regenerated fixture suite. The I/O assertion intentionally avoids pinning UNTR seen-entry counts because Git versions serialize equivalent cache contents differently. The full Git-compatible `Location ..., system ...` identifier prevents copied indexes from reusing filesystem stats on another system. Windows compares native canonical paths because Git for Windows canonicalizes worktree spelling through the OS. Validated with GIX_TEST_IGNORE_ARCHIVES=1 just nextest -p gix-dir, SHA-1/SHA-256 and Windows feature checks, and clippy for all gix-dir targets and features. --- Cargo.lock | 2 + gix-dir/Cargo.toml | 9 + gix-dir/benches/dirwalk.rs | 172 +++++++++++++ gix-dir/src/walk/function.rs | 8 + gix-dir/src/walk/mod.rs | 8 + gix-dir/src/walk/readdir.rs | 240 +++++++++++++----- gix-dir/src/walk/untracked_cache.rs | 194 ++++++++++++++ gix-dir/tests/dir/walk.rs | 115 +++++++-- .../fixtures/generated-archives/.gitignore | 2 + gix-dir/tests/fixtures/many-symlinks.sh | 10 + gix-dir/tests/fixtures/many.sh | 31 +++ gix-dir/tests/walk_utils/mod.rs | 7 +- 12 files changed, 713 insertions(+), 85 deletions(-) create mode 100644 gix-dir/benches/dirwalk.rs create mode 100644 gix-dir/src/walk/untracked_cache.rs diff --git a/Cargo.lock b/Cargo.lock index c5964c2b42d..42d570f032e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1839,6 +1839,7 @@ name = "gix-dir" version = "0.28.0" dependencies = [ "bstr", + "criterion", "gix-discover", "gix-fs", "gix-ignore", @@ -1851,6 +1852,7 @@ dependencies = [ "gix-utils", "gix-worktree", "pretty_assertions", + "rustix", "thiserror 2.0.18", ] diff --git a/gix-dir/Cargo.toml b/gix-dir/Cargo.toml index 7b69120cff1..ee45b9c2d00 100644 --- a/gix-dir/Cargo.toml +++ b/gix-dir/Cargo.toml @@ -15,6 +15,11 @@ include = ["/src/**/*", "/LICENSE-*"] doctest = false test = false +[[bench]] +name = "dirwalk" +harness = false +path = "./benches/dirwalk.rs" + [features] ## Enable support for the SHA-1 hash by forwarding the feature to dependencies. sha1 = ["gix-index/sha1"] @@ -36,7 +41,11 @@ gix-utils = { version = "^0.3.5", path = "../gix-utils", features = ["bstr"] } bstr = { version = "1.12.0", default-features = false } thiserror = "2.0.18" +[target.'cfg(unix)'.dependencies] +rustix = { version = "1.1.2", default-features = false, features = ["system"] } + [dev-dependencies] +criterion = "0.8.2" gix-testtools = { path = "../tests/tools" } gix-fs = { path = "../gix-fs" } pretty_assertions = "1.4.0" diff --git a/gix-dir/benches/dirwalk.rs b/gix-dir/benches/dirwalk.rs new file mode 100644 index 00000000000..4b9a6ca8144 --- /dev/null +++ b/gix-dir/benches/dirwalk.rs @@ -0,0 +1,172 @@ +use std::{hint::black_box, path::PathBuf}; + +use bstr::ByteSlice; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use gix_dir::walk; +use gix_testtools::FixtureState; + +fn dirwalk(c: &mut Criterion) { + let fixture = gix_testtools::rust_fixture_read_only("dirwalk-benchmark", 2, |state| { + if let FixtureState::Uninitialized(root) = state { + create_clean_flat(&root.join("clean-flat"))?; + create_clean_wide(&root.join("clean-wide"))?; + create_untracked_wide(&root.join("untracked-wide"))?; + } + Ok(()) + }) + .expect("benchmark fixture can be created") + .0; + + let mut group = c.benchmark_group("dirwalk"); + for name in ["clean-flat", "clean-wide", "untracked-wide"] { + let scenario = Scenario::new(fixture.join(name)); + group.bench_with_input(BenchmarkId::from_parameter(name), &scenario, |b, scenario| { + b.iter(|| black_box(scenario.walk())); + }); + } +} + +criterion_group!(benches, dirwalk); +criterion_main!(benches); + +struct Scenario { + root: PathBuf, + git_dir_realpath: PathBuf, + index: gix_index::State, +} + +impl Scenario { + fn new(root: PathBuf) -> Self { + let git_dir = root.join(".git"); + let index = std::fs::read(git_dir.join("index")) + .map_err(|err| format!("cannot read benchmark index: {err}")) + .and_then(|bytes| { + gix_index::State::from_bytes( + &bytes, + std::time::UNIX_EPOCH.into(), + gix_index::hash::Kind::Sha1, + Default::default(), + ) + .map(|(index, _)| index) + .map_err(|err| format!("cannot decode benchmark index: {err}")) + }) + .expect("Git creates a valid benchmark index"); + assert!(index.untracked().is_some(), "Git must populate the UNTR cache"); + Scenario { + git_dir_realpath: gix_path::realpath(&git_dir).expect("git directory can be resolved"), + root, + index, + } + } + + fn walk(&self) -> walk::Outcome { + let mut pathspec = gix_pathspec::Search::from_specs( + std::iter::empty::(), + None, + "benchmark has no absolute pathspecs".as_ref(), + ) + .expect("empty pathspec is valid"); + let mut excludes = gix_worktree::Stack::from_state_and_ignore_case( + &self.root, + false, + gix_worktree::stack::State::IgnoreStack(gix_worktree::stack::state::Ignore::new( + Default::default(), + Default::default(), + None, + gix_worktree::stack::state::ignore::Source::WorktreeThenIdMappingIfNotSkipped, + Default::default(), + )), + &self.index, + self.index.path_backing(), + ); + let mut delegate = Ignore; + walk( + &self.root, + walk::Context { + should_interrupt: None, + git_dir_realpath: &self.git_dir_realpath, + current_dir: &self.root, + index: &self.index, + ignore_case_index_lookup: None, + pathspec: &mut pathspec, + pathspec_attributes: &mut |_, _, _, _| unreachable!("benchmark pathspecs have no attributes"), + excludes: Some(&mut excludes), + objects: &gix_object::find::Never, + explicit_traversal_root: None, + }, + walk::Options { + use_untracked_cache: true, + emit_untracked: walk::EmissionMode::CollapseDirectory, + ..Default::default() + }, + &mut delegate, + ) + .expect("benchmark dirwalk succeeds") + .0 + } +} + +struct Ignore; + +impl walk::Delegate for Ignore { + fn emit( + &mut self, + _entry: gix_dir::EntryRef<'_>, + _collapsed_directory_status: Option, + ) -> walk::Action { + std::ops::ControlFlow::Continue(()) + } +} + +fn create_clean_flat(root: &std::path::Path) -> gix_testtools::Result { + init(root)?; + for file_idx in 0..10_000 { + std::fs::write(root.join(format!("file-{file_idx:05}")), [])?; + } + commit_and_prime(root) +} + +fn create_clean_wide(root: &std::path::Path) -> gix_testtools::Result { + init(root)?; + create_wide_tree(root)?; + commit_and_prime(root) +} + +fn create_untracked_wide(root: &std::path::Path) -> gix_testtools::Result { + init(root)?; + std::fs::write(root.join("tracked"), [])?; + commit_and_prime(root)?; + create_wide_tree(root)?; + prime(root) +} + +fn create_wide_tree(root: &std::path::Path) -> gix_testtools::Result { + for dir_idx in 0..100 { + let dir = root.join(format!("dir-{dir_idx:03}")); + std::fs::create_dir(&dir)?; + for file_idx in 0..100 { + std::fs::write(dir.join(format!("file-{file_idx:03}")), [])?; + } + } + Ok(()) +} + +fn init(root: &std::path::Path) -> gix_testtools::Result { + std::fs::create_dir(root)?; + gix_testtools::git(root, "init --quiet")?; + gix_testtools::git(root, "config core.untrackedCache true")?; + gix_testtools::git(root, "config core.excludesFile .git/no-global-excludes")?; + Ok(()) +} + +fn commit_and_prime(root: &std::path::Path) -> gix_testtools::Result { + gix_testtools::git(root, "add .")?; + gix_testtools::git(root, "commit --quiet -m baseline")?; + prime(root) +} + +fn prime(root: &std::path::Path) -> gix_testtools::Result { + let status = gix_testtools::git(root, "status --porcelain")?; + black_box(status.as_bytes().as_bstr()); + Ok(()) +} diff --git a/gix-dir/src/walk/function.rs b/gix-dir/src/walk/function.rs index 674737ff20b..49ed7d133fd 100644 --- a/gix-dir/src/walk/function.rs +++ b/gix-dir/src/walk/function.rs @@ -104,6 +104,13 @@ pub fn walk( return Ok((out, root.to_owned())); } + let untracked_cache = crate::walk::untracked_cache::State::new( + worktree_root, + ctx.index, + ctx.pathspec, + ctx.explicit_traversal_root, + options, + ); let mut state = readdir::State::new(worktree_root, ctx.current_dir, options.for_deletion.is_some()); let may_collapse = root != worktree_root && state.may_collapse(¤t); let (action, _) = readdir::recursive( @@ -116,6 +123,7 @@ pub fn walk( delegate, &mut out, &mut state, + untracked_cache.as_ref().map(|cache| (cache, 0)), )?; if action.is_continue() { state.emit_remaining(may_collapse, options, &mut out, delegate); diff --git a/gix-dir/src/walk/mod.rs b/gix-dir/src/walk/mod.rs index 0529f097cca..d8dc2769826 100644 --- a/gix-dir/src/walk/mod.rs +++ b/gix-dir/src/walk/mod.rs @@ -142,6 +142,13 @@ pub enum ForDeletionMode { /// Options for use in [`walk()`](function::walk()) function. #[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] pub struct Options<'a> { + /// If `true`, use a compatible `UNTR` index extension to avoid reading unchanged directories. + /// + /// Callers must only enable this when Git configuration permits use of the cache. + /// The cache is otherwise validated against the worktree and configured exclude files before use. + pub use_untracked_cache: bool, + /// The effective `core.excludesFile`, including Git's user-level default, to validate against the `UNTR` extension. + pub untracked_cache_excludes_file: Option<&'a std::path::Path>, /// If `true`, the filesystem will store paths as decomposed unicode, i.e. `ä` becomes `"a\u{308}"`, which means that /// we have to turn these forms back from decomposed to precomposed unicode before storing it in the index or generally /// using it. This also applies to input received from the command-line, so callers may have to be aware of this and @@ -308,3 +315,4 @@ pub enum Error { mod classify; pub(crate) mod function; mod readdir; +mod untracked_cache; diff --git a/gix-dir/src/walk/readdir.rs b/gix-dir/src/walk/readdir.rs index 56374cb2f86..422ce73d00b 100644 --- a/gix-dir/src/walk/readdir.rs +++ b/gix-dir/src/walk/readdir.rs @@ -15,6 +15,7 @@ use crate::{ EmissionMode::CollapseDirectory, Error, ForDeletionMode, Options, Outcome, classify, function::{can_recurse, emit_entry}, + untracked_cache, }, }; @@ -32,96 +33,120 @@ pub(super) fn recursive( delegate: &mut dyn Delegate, out: &mut Outcome, state: &mut State, + cache_dir: Option<(&untracked_cache::State<'_>, usize)>, ) -> Result<(Action, bool), Error> { if ctx.should_interrupt.is_some_and(|flag| flag.load(Ordering::Relaxed)) { return Err(Error::Interrupted); } - out.read_dir_calls += 1; - let entries = gix_fs::read_dir(current, opts.precompose_unicode).map_err(|err| Error::ReadDir { - path: current.to_owned(), - source: err, - })?; let mut num_entries = 0; let mark = state.mark(may_collapse); let mut prevent_collapse = false; - for entry in entries { - let entry = entry.map_err(|err| Error::DirEntry { - parent_directory: current.to_owned(), - source: err, - })?; - // Important to count right away, otherwise the directory could be seen as empty even though it's not. - // That is, this should be independent of the kind. - num_entries += 1; - - let prev_len = current_bstr.len(); - if prev_len != 0 { - current_bstr.push(b'/'); - } - let file_name = entry.file_name(); - current_bstr.extend_from_slice( - gix_path::try_os_str_into_bstr(Cow::Borrowed(file_name.as_ref())) - .expect("no illformed UTF-8") - .as_ref(), - ); - current.push(file_name); - let mut info = classify::path( - current, - current_bstr, - if prev_len == 0 { 0 } else { prev_len + 1 }, - None, - || entry.file_type().ok().map(Into::into), - opts, - ctx, - )?; - - if can_recurse( - current_bstr.as_bstr(), - info, - opts.for_deletion, - false, /* is root */ - delegate, - ) { - let subdir_may_collapse = state.may_collapse(current); - let (action, subdir_prevent_collapse) = recursive( - subdir_may_collapse, + let cached = cache_dir.and_then(|(cache, index)| { + cache + .directory(index, current, current_bstr.as_bstr(), current_info, ctx) + .map(|directory| (cache, index, directory)) + }); + if let Some((cache, directory_index, directory)) = cached { + let has_tracked_descendant = + untracked_cache::has_tracked_descendant(current_bstr.as_bstr(), opts.ignore_case, ctx); + prevent_collapse = has_tracked_descendant; + num_entries += usize::from(has_tracked_descendant); + for child_index in directory.sub_directories() { + let name = cache + .child_name(*child_index) + .expect("UNTR child indices were validated during decoding"); + num_entries += 1; + let (action, child_prevent_collapse) = visit( + name, + Some(entry::Kind::Directory), + || None, + false, + Some((cache, *child_index)), current, current_bstr, - info, ctx, opts, delegate, out, state, )?; - prevent_collapse |= subdir_prevent_collapse; + prevent_collapse |= child_prevent_collapse; if action.is_break() { return Ok((action, prevent_collapse)); } - } else { - if opts.for_deletion == Some(ForDeletionMode::IgnoredDirectoriesCanHideNestedRepositories) - && info.disk_kind == Some(entry::Kind::Directory) - && matches!(info.status, Status::Ignored(_)) - { - info.disk_kind = classify::maybe_upgrade_to_repository( - info.disk_kind, - true, - false, - current, - ctx.current_dir, - ctx.git_dir_realpath, - ); + } + for name in directory.untracked_entries() { + let is_directory = name.ends_with_str("/"); + let name = name.strip_suffix(b"/").unwrap_or(name.as_bstr()).as_bstr(); + if is_directory && cache.child_index(directory_index, name).is_some() { + continue; } - if !state.held_for_directory_collapse(current_bstr.as_bstr(), info, &opts) { - let action = emit_entry(Cow::Borrowed(current_bstr.as_bstr()), info, None, opts, out, delegate); - if action.is_break() { - return Ok((action, prevent_collapse)); - } + num_entries += 1; + let on_demand_disk_kind = (!is_directory) + .then(|| { + current + .join(gix_path::from_bstr(name)) + .symlink_metadata() + .ok() + .map(|metadata| metadata.file_type().into()) + }) + .flatten(); + let (action, child_prevent_collapse) = visit( + name, + is_directory.then_some(entry::Kind::Directory), + || on_demand_disk_kind, + is_directory, + None, + current, + current_bstr, + ctx, + opts, + delegate, + out, + state, + )?; + prevent_collapse |= child_prevent_collapse; + if action.is_break() { + return Ok((action, prevent_collapse)); + } + } + } else { + out.read_dir_calls += 1; + let entries = gix_fs::read_dir(current, opts.precompose_unicode).map_err(|err| Error::ReadDir { + path: current.to_owned(), + source: err, + })?; + for entry in entries { + let entry = entry.map_err(|err| Error::DirEntry { + parent_directory: current.to_owned(), + source: err, + })?; + // Count before classification so unreadable entries still keep a directory from appearing empty. + num_entries += 1; + let file_name = gix_path::try_os_str_into_bstr(Cow::Borrowed(entry.file_name().as_ref())) + .expect("no illformed UTF-8") + .into_owned(); + let (action, child_prevent_collapse) = visit( + file_name.as_bstr(), + None, + || entry.file_type().ok().map(Into::into), + false, + None, + current, + current_bstr, + ctx, + opts, + delegate, + out, + state, + )?; + prevent_collapse |= child_prevent_collapse; + if action.is_break() { + return Ok((action, prevent_collapse)); } } - current_bstr.truncate(prev_len); - current.pop(); } let res = mark.reduce_held_entries( @@ -139,6 +164,87 @@ pub(super) fn recursive( Ok((res, prevent_collapse)) } +#[expect(clippy::too_many_arguments)] +fn visit( + name: &BStr, + disk_kind: Option, + on_demand_disk_kind: impl FnOnce() -> Option, + cached_leaf_directory: bool, + cache_dir: Option<(&untracked_cache::State<'_>, usize)>, + current: &mut PathBuf, + current_bstr: &mut BString, + ctx: &mut Context<'_>, + opts: Options<'_>, + delegate: &mut dyn Delegate, + out: &mut Outcome, + state: &mut State, +) -> Result<(Action, bool), Error> { + let prev_len = current_bstr.len(); + if prev_len != 0 { + current_bstr.push(b'/'); + } + current_bstr.extend_from_slice(name); + current.push(gix_path::from_bstr(name)); + + let mut info = classify::path( + current, + current_bstr, + if prev_len == 0 { 0 } else { prev_len + 1 }, + disk_kind, + on_demand_disk_kind, + opts, + ctx, + )?; + + let mut prevent_collapse = false; + let action = if !cached_leaf_directory + && can_recurse( + current_bstr.as_bstr(), + info, + opts.for_deletion, + false, /* is root */ + delegate, + ) { + let subdir_may_collapse = state.may_collapse(current); + let (action, subdir_prevent_collapse) = recursive( + subdir_may_collapse, + current, + current_bstr, + info, + ctx, + opts, + delegate, + out, + state, + cache_dir, + )?; + prevent_collapse = subdir_prevent_collapse; + action + } else { + if opts.for_deletion == Some(ForDeletionMode::IgnoredDirectoriesCanHideNestedRepositories) + && info.disk_kind == Some(entry::Kind::Directory) + && matches!(info.status, Status::Ignored(_)) + { + info.disk_kind = classify::maybe_upgrade_to_repository( + info.disk_kind, + true, + false, + current, + ctx.current_dir, + ctx.git_dir_realpath, + ); + } + if state.held_for_directory_collapse(current_bstr.as_bstr(), info, &opts) { + std::ops::ControlFlow::Continue(()) + } else { + emit_entry(Cow::Borrowed(current_bstr.as_bstr()), info, None, opts, out, delegate) + } + }; + current_bstr.truncate(prev_len); + current.pop(); + Ok((action, prevent_collapse)) +} + pub(super) struct State { /// The entries to hold back until it's clear what to do with them. pub on_hold: Vec, diff --git a/gix-dir/src/walk/untracked_cache.rs b/gix-dir/src/walk/untracked_cache.rs new file mode 100644 index 00000000000..84dae9c11be --- /dev/null +++ b/gix-dir/src/walk/untracked_cache.rs @@ -0,0 +1,194 @@ +use std::path::Path; + +use bstr::{BStr, BString, ByteSlice}; + +use crate::walk::{Context, EmissionMode, Options, classify}; + +const DIR_SHOW_OTHER_DIRECTORIES: u32 = 1 << 1; +const DIR_HIDE_EMPTY_DIRECTORIES: u32 = 1 << 2; + +pub(super) struct State<'a> { + cache: &'a gix_index::extension::UntrackedCache, +} + +impl<'a> State<'a> { + pub(super) fn new( + worktree_root: &Path, + index: &'a gix_index::State, + pathspec: &gix_pathspec::Search, + explicit_traversal_root: Option<&Path>, + opts: Options<'_>, + ) -> Option { + let cache = index.untracked()?; + (opts.use_untracked_cache + && opts.emit_untracked == EmissionMode::CollapseDirectory + && opts.emit_ignored.is_none() + && opts.for_deletion.is_none() + && !opts.recurse_repositories + && !opts.classify_untracked_bare_repositories + && !opts.emit_tracked + && !opts.emit_empty_directories + && opts.emit_collapsed.is_none() + && pathspec.patterns().len() == 0 + && explicit_traversal_root.is_none_or(|root| root == worktree_root) + && cache.dir_flags() == DIR_SHOW_OTHER_DIRECTORIES | DIR_HIDE_EMPTY_DIRECTORIES + && cache.exclude_filename_per_dir() == ".gitignore" + && opts.untracked_cache_excludes_file.map_or_else( + || cache.excludes_file().is_none().then_some(true), + |path| { + ignore_oid_at_path_matches( + path, + index.object_hash(), + cache + .excludes_file() + .map(gix_index::extension::untracked_cache::OidStat::id), + ) + }, + )? + && identifier_matches(cache.identifier().as_bstr(), worktree_root)? + && ignore_oid_at_path_matches( + &worktree_root.join(".git/info/exclude"), + index.object_hash(), + cache + .info_exclude() + .map(gix_index::extension::untracked_cache::OidStat::id), + )?) + .then_some(State { cache }) + } + + pub(super) fn directory( + &self, + index: usize, + current: &Path, + current_rela_path: &BStr, + current_info: classify::Outcome, + ctx: &Context<'_>, + ) -> Option<&'a gix_index::extension::untracked_cache::Directory> { + let directory = self.cache.directories().get(index)?; + let metadata = gix_index::fs::Metadata::from_path_no_follow(current).ok()?; + let stat = gix_index::entry::Stat::from_fs(&metadata).ok()?; + let expected_check_only = + !current_rela_path.is_empty() && current_info.status == crate::entry::Status::Untracked; + let cached_stat = directory.stat()?; + let stat_options = gix_index::entry::stat::Options { + use_nsec: cached_stat.mtime.nsecs != 0, + ..Default::default() + }; + (cached_stat.matches(&stat, stat_options) + && directory.check_only() == expected_check_only + && ignore_oid_matches(current, current_rela_path, ctx, directory.exclude_file_oid())?) + .then_some(directory) + } + + pub(super) fn child_index(&self, directory_index: usize, name: &BStr) -> Option { + self.cache + .directories() + .get(directory_index)? + .sub_directories() + .iter() + .copied() + .find(|index| { + self.cache + .directories() + .get(*index) + .is_some_and(|directory| directory.name() == name) + }) + } + + pub(super) fn child_name(&self, index: usize) -> Option<&'a BStr> { + self.cache + .directories() + .get(index) + .map(gix_index::extension::untracked_cache::Directory::name) + } +} + +#[cfg(windows)] +fn identifier_matches(identifier: &BStr, worktree_root: &Path) -> Option { + let location = identifier + .strip_prefix(b"Location ")? + .strip_suffix(b", system Windows\0")?; + Some( + std::fs::canonicalize(gix_path::from_bstr(location.as_bstr())).ok()? + == std::fs::canonicalize(worktree_root).ok()?, + ) +} + +#[cfg(not(windows))] +fn identifier_matches(identifier: &BStr, worktree_root: &Path) -> Option { + let worktree_location = gix_path::into_bstr(gix_path::realpath(worktree_root).ok()?); + Some(identifier == format!("Location {}, system {}\0", worktree_location, system_name()?)) +} + +#[cfg(unix)] +fn system_name() -> Option { + rustix::system::uname().sysname().to_str().ok().map(ToOwned::to_owned) +} + +#[cfg(not(any(unix, windows)))] +fn system_name() -> Option { + None +} + +pub(super) fn has_tracked_descendant(directory: &BStr, ignore_case: bool, ctx: &Context<'_>) -> bool { + ctx.ignore_case_index_lookup + .map_or_else( + || ctx.index.entry_closest_to_directory_or_directory(directory), + |lookup| { + ctx.index + .entry_closest_to_directory_or_directory_icase(directory, ignore_case, lookup) + }, + ) + .is_some() +} + +fn ignore_oid_matches( + current: &Path, + current_rela_path: &BStr, + ctx: &Context<'_>, + expected: Option, +) -> Option { + let ignore_path = current.join(".gitignore"); + match std::fs::read(&ignore_path) { + Ok(mut data) => { + let raw = gix_object::compute_hash(ctx.index.object_hash(), gix_object::Kind::Blob, &data).ok()?; + if Some(raw) == expected { + return Some(true); + } + data.push(b'\n'); + gix_object::compute_hash(ctx.index.object_hash(), gix_object::Kind::Blob, &data) + .ok() + .map(|id| Some(id) == expected) + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + let mut rela_path = BString::from(current_rela_path); + if !rela_path.is_empty() { + rela_path.push(b'/'); + } + rela_path.extend_from_slice(b".gitignore"); + Some(ctx.index.entry_by_path(rela_path.as_bstr()).map(|entry| entry.id) == expected) + } + Err(_) => None, + } +} + +fn ignore_oid_at_path_matches( + path: &Path, + object_hash: gix_index::hash::Kind, + expected: Option, +) -> Option { + match std::fs::read(path) { + Ok(mut data) => { + let raw = gix_object::compute_hash(object_hash, gix_object::Kind::Blob, &data).ok()?; + if Some(raw) == expected { + return Some(true); + } + data.push(b'\n'); + gix_object::compute_hash(object_hash, gix_object::Kind::Blob, &data) + .ok() + .map(|id| Some(id) == expected) + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Some(expected.is_none()), + Err(_) => None, + } +} diff --git a/gix-dir/tests/dir/walk.rs b/gix-dir/tests/dir/walk.rs index d4c2a7f9830..c700384608a 100644 --- a/gix-dir/tests/dir/walk.rs +++ b/gix-dir/tests/dir/walk.rs @@ -19,6 +19,47 @@ use crate::walk_utils::{ try_collect_filtered_opts, try_collect_filtered_opts_collect, try_collect_filtered_opts_collect_with_root, }; +#[test] +fn untracked_cache_avoids_reading_unchanged_directories() { + let root = fixture("only-untracked"); + let mut without_cache_options = options(); + without_cache_options.use_untracked_cache = false; + without_cache_options.emit_untracked = CollapseDirectory; + let ((without_cache, _), expected) = + collect(&root, None, |keep, ctx| walk(&root, ctx, without_cache_options, keep)); + + let mut with_cache_options = options(); + with_cache_options.emit_untracked = CollapseDirectory; + let ((with_cache, _), actual) = collect(&root, None, |keep, ctx| walk(&root, ctx, with_cache_options, keep)); + + assert_eq!(actual, expected, "UNTR replay must produce the same entries"); + assert!( + with_cache.read_dir_calls < without_cache.read_dir_calls, + "UNTR replay should reduce directory reads, got {} with cache and {} without", + with_cache.read_dir_calls, + without_cache.read_dir_calls + ); +} + +#[test] +fn untracked_cache_does_not_replay_descendants_after_inherited_ignore_changes() { + let root = fixture("untracked-cache-changed-parent-ignore"); + let mut without_cache_options = options(); + without_cache_options.use_untracked_cache = false; + without_cache_options.emit_untracked = CollapseDirectory; + let (_, expected) = collect(&root, None, |keep, ctx| walk(&root, ctx, without_cache_options, keep)); + + let mut with_cache_options = options(); + with_cache_options.emit_untracked = CollapseDirectory; + let (_, actual) = collect(&root, None, |keep, ctx| walk(&root, ctx, with_cache_options, keep)); + + assert_eq!( + actual, expected, + "rejecting a parent cache must invalidate descendants that inherited its excludes" + ); + assert_eq!(actual, [entry("child", Untracked, Directory)]); +} + #[test] #[cfg(unix)] fn root_is_fifo() { @@ -1056,9 +1097,9 @@ fn only_untracked_with_prefix_deletion() -> crate::Result { #[test] fn only_untracked() -> crate::Result { let root = fixture("only-untracked"); - let ((out, _root), entries) = collect(&root, None, |keep, ctx| walk(&root, ctx, options(), keep)); + let ((without_cache, _root), entries) = collect(&root, None, |keep, ctx| walk(&root, ctx, options(), keep)); assert_eq!( - out, + without_cache, walk::Outcome { read_dir_calls: 3, returned_entries: entries.len(), @@ -1108,13 +1149,12 @@ fn only_untracked() -> crate::Result { ) }); assert_eq!( - out, - walk::Outcome { - read_dir_calls: 3, - returned_entries: entries.len(), - seen_entries: 7 + 2, - }, - "There are 2 extra directories that we fold into, but ultimately discard" + out.read_dir_calls, 0, + "UNTR supplies the cached entries without opening directories" + ); + assert!( + out.read_dir_calls < without_cache.read_dir_calls, + "UNTR should reduce directory reads" ); assert_eq!( entries, @@ -1331,9 +1371,9 @@ fn expendable_and_precious() { assert_eq!( out, walk::Outcome { - read_dir_calls: 6, + read_dir_calls: 0, returned_entries: entries.len(), - seen_entries: 16 + 2, + seen_entries: 8, } ); @@ -1395,11 +1435,11 @@ fn subdir_untracked() -> crate::Result { assert_eq!( out, walk::Outcome { - read_dir_calls: 3, + read_dir_calls: 0, returned_entries: entries.len(), - seen_entries: 7 + 1, + seen_entries: 2, }, - "there is a folded directory we added" + "UNTR supplies only the entries needed to produce the folded directory" ); assert_eq!(entries, [entry("d/d", Untracked, Directory)]); Ok(()) @@ -1966,11 +2006,11 @@ fn untracked_and_ignored() -> crate::Result { assert_eq!( out, walk::Outcome { - read_dir_calls: 5, + read_dir_calls: 0, returned_entries: entries.len(), - seen_entries: 21 + 1, + seen_entries: 5, }, - "we still encounter the same amount of entries, and 1 folded directory" + "UNTR supplies only the entries needed for aggregation" ); assert_eq!( entries, @@ -4407,6 +4447,47 @@ fn type_mismatch_ignore_case() { ); } +#[test] +fn untracked_cache_honors_case_insensitive_tracked_directories() { + let root = fixture("untracked-cache-icase"); + let ((out, _root), entries) = try_collect_filtered_opts_collect( + &root, + None, + |keep, ctx| { + walk( + &root, + ctx, + walk::Options { + emit_untracked: CollapseDirectory, + ignore_case: true, + ..options() + }, + keep, + ) + }, + None::<&str>, + Options { + fresh_index: false, + ..Default::default() + }, + ) + .expect("success"); + assert_eq!( + out, + walk::Outcome { + read_dir_calls: 0, + returned_entries: 1, + seen_entries: 1, + }, + "the cached directory and its tracked contents are accounted for without filesystem reads" + ); + assert_eq!( + entries, + [entry("dir/untracked", Untracked, File)], + "case-folded tracked contents prevent collapsing their untracked sibling" + ); +} + #[test] fn type_mismatch_ignore_case_clash_dir_is_file() { let root = fixture("type-mismatch-icase-clash-dir-is-file"); diff --git a/gix-dir/tests/fixtures/generated-archives/.gitignore b/gix-dir/tests/fixtures/generated-archives/.gitignore index 5dede136750..3f483fe26a4 100644 --- a/gix-dir/tests/fixtures/generated-archives/.gitignore +++ b/gix-dir/tests/fixtures/generated-archives/.gitignore @@ -12,3 +12,5 @@ many.tar many-symlinks.tar # Uses `mkfifo`; FIFOs cannot be represented in archives extracted on Windows. fifo.tar +# Synthetic benchmark repositories are large and cheap to regenerate. +rust-dirwalk-benchmark.tar diff --git a/gix-dir/tests/fixtures/many-symlinks.sh b/gix-dir/tests/fixtures/many-symlinks.sh index a9fe584da2f..d8c939763f9 100755 --- a/gix-dir/tests/fixtures/many-symlinks.sh +++ b/gix-dir/tests/fixtures/many-symlinks.sh @@ -66,3 +66,13 @@ git init -q submodule-symlink rm -Rf sub ln -s ../module sub ) + +# Populate UNTR caches so every directory-walk test exercises cache eligibility. +find . -name .git -prune | while read -r git_dir +do + repo=${git_dir%/.git} + git -C "$repo" config core.untrackedCache true + git -C "$repo" config core.excludesFile .git/no-global-excludes + git -C "$repo" update-index --untracked-cache --index-version 2 + git -C "$repo" status --porcelain --ignore-submodules=all >/dev/null +done diff --git a/gix-dir/tests/fixtures/many.sh b/gix-dir/tests/fixtures/many.sh index 4d8850b616a..0f522126d21 100755 --- a/gix-dir/tests/fixtures/many.sh +++ b/gix-dir/tests/fixtures/many.sh @@ -183,6 +183,15 @@ git init only-untracked >c ) +git init untracked-cache-changed-parent-ignore +(cd untracked-cache-changed-parent-ignore + mkdir child + echo ignored >.gitignore + >child/ignored + git add .gitignore + git commit -m "track inherited ignore" +) + git init ignored-with-empty (cd ignored-with-empty echo "/target/" >> .gitignore @@ -328,6 +337,15 @@ git init type-mismatch-icase rm file-is-dir && mkdir File-is-Dir && >File-is-Dir/b ) +git init untracked-cache-icase +(cd untracked-cache-icase + mkdir Dir && >Dir/tracked + git add . && git commit -m "tracked directory" + mv Dir dir + >dir/untracked + git config core.ignoreCase true +) + git init type-mismatch-icase-clash-dir-is-file (cd type-mismatch-icase-clash-dir-is-file empty_oid=$(git hash-object -w --stdin /dev/null +done + +# Make the root cache stale while leaving its child's stat and local excludes unchanged. +>untracked-cache-changed-parent-ignore/.gitignore diff --git a/gix-dir/tests/walk_utils/mod.rs b/gix-dir/tests/walk_utils/mod.rs index 93196477e65..689b9f78d1b 100644 --- a/gix-dir/tests/walk_utils/mod.rs +++ b/gix-dir/tests/walk_utils/mod.rs @@ -18,12 +18,17 @@ pub fn fixture(name: &str) -> PathBuf { /// Default options pub fn options() -> walk::Options<'static> { - walk::Options::default() + walk::Options { + use_untracked_cache: true, + ..Default::default() + } } /// Default options pub fn options_emit_all() -> walk::Options<'static> { walk::Options { + use_untracked_cache: true, + untracked_cache_excludes_file: None, precompose_unicode: false, ignore_case: false, recurse_repositories: false, From cf80446c1cd6db190939731c974c2535c7c33fdc Mon Sep 17 00:00:00 2001 From: Codex GPT-5 Date: Sat, 25 Jul 2026 07:50:32 +0200 Subject: [PATCH 2/2] feat: use configured UNTR caches for status dirwalks. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Honor core.untrackedCache for repository dirwalks and status. Treat false as the explicit opt-out while retaining an existing cache for true, keep, unset, or unrecognized values, matching Git’s permissive behavior. Pass the effective global excludes file to gix-dir so cache validation observes the same ignore inputs as normal traversal. Add a status regression proving enabled and keep modes reduce directory reads, false disables the optimization, and changed excludes invalidate it. Validation: - just nextest -p gix - cargo clippy -p gix --all-targets -- -D warnings - cargo check -p gix --no-default-features --features status,sha1 --- gix/src/config/cache/access.rs | 18 +++++++--- gix/src/config/tree/sections/core.rs | 3 ++ gix/src/dirwalk/options.rs | 2 ++ gix/src/repository/dirwalk.rs | 3 ++ gix/src/status/index_worktree.rs | 11 +++++- gix/tests/gix/status.rs | 54 ++++++++++++++++++++++++++++ 6 files changed, 86 insertions(+), 5 deletions(-) diff --git a/gix/src/config/cache/access.rs b/gix/src/config/cache/access.rs index 4c9357d9c2a..16da2845067 100644 --- a/gix/src/config/cache/access.rs +++ b/gix/src/config/cache/access.rs @@ -247,6 +247,19 @@ impl Cache { self.trusted_file_path(Core::EXCLUDES_FILE) } + #[cfg(feature = "excludes")] + pub(crate) fn effective_excludes_file(&self) -> Result, config::exclude_stack::Error> { + Ok(match self.excludes_file()? { + Some(user_path) => Some(user_path), + None => self.xdg_config_path("ignore")?, + }) + } + + #[cfg(feature = "dirwalk")] + pub(crate) fn use_untracked_cache(&self) -> bool { + self.resolved.boolean(Core::UNTRACKED_CACHE).ok().flatten() != Some(false) + } + /// A helper to obtain a file from trusted configuration at `section_name`, `subsection_name`, and `key`, which is interpolated /// if present. pub(crate) fn trusted_file_path( @@ -393,10 +406,7 @@ impl Cache { source: gix_worktree::stack::state::ignore::Source, buf: &mut Vec, ) -> Result { - let excludes_file = match self.excludes_file()? { - Some(user_path) => Some(user_path), - None => self.xdg_config_path("ignore")?, - }; + let excludes_file = self.effective_excludes_file()?; let parse_ignore = self.ignore_pattern_parser()?; Ok(gix_worktree::stack::state::Ignore::new( overrides.unwrap_or_default(), diff --git a/gix/src/config/tree/sections/core.rs b/gix/src/config/tree/sections/core.rs index 241f7eb04bd..08fe3cb7db3 100644 --- a/gix/src/config/tree/sections/core.rs +++ b/gix/src/config/tree/sections/core.rs @@ -62,6 +62,8 @@ impl Core { pub const SYMLINKS: keys::Boolean = keys::Boolean::new_boolean("symlinks", &config::Tree::CORE); /// The `core.trustCTime` key. pub const TRUST_C_TIME: keys::Boolean = keys::Boolean::new_boolean("trustCTime", &config::Tree::CORE); + /// The `core.untrackedCache` key. + pub const UNTRACKED_CACHE: keys::Any = keys::Any::new("untrackedCache", &config::Tree::CORE); /// The `core.worktree` key. pub const WORKTREE: keys::Any = keys::Any::new("worktree", &config::Tree::CORE) .with_environment_override("GIT_WORK_TREE") @@ -129,6 +131,7 @@ impl Section for Core { &Self::REPOSITORY_FORMAT_VERSION, &Self::SYMLINKS, &Self::TRUST_C_TIME, + &Self::UNTRACKED_CACHE, &Self::WORKTREE, &Self::PROTECT_HFS, &Self::PROTECT_NTFS, diff --git a/gix/src/dirwalk/options.rs b/gix/src/dirwalk/options.rs index 8c292821719..4628368c04b 100644 --- a/gix/src/dirwalk/options.rs +++ b/gix/src/dirwalk/options.rs @@ -26,6 +26,8 @@ impl Options { impl From for gix_dir::walk::Options<'static> { fn from(v: Options) -> Self { gix_dir::walk::Options { + use_untracked_cache: false, + untracked_cache_excludes_file: None, precompose_unicode: v.precompose_unicode, ignore_case: v.ignore_case, recurse_repositories: v.recurse_repositories, diff --git a/gix/src/repository/dirwalk.rs b/gix/src/repository/dirwalk.rs index eb8e4b6e5e7..b111b0831da 100644 --- a/gix/src/repository/dirwalk.rs +++ b/gix/src/repository/dirwalk.rs @@ -59,6 +59,9 @@ impl Repository { let fs_caps = self.filesystem_options()?; let accelerate_lookup = fs_caps.ignore_case.then(|| index.prepare_icase_backing()); let mut opts = gix_dir::walk::Options::from(options); + let excludes_file = self.config.effective_excludes_file()?; + opts.use_untracked_cache = self.config.use_untracked_cache(); + opts.untracked_cache_excludes_file = excludes_file.as_deref(); let worktree_relative_worktree_dirs_storage; if let Some(workdir) = self.workdir().filter(|_| opts.for_deletion.is_some()) { let linked_worktrees = self.worktrees()?; diff --git a/gix/src/status/index_worktree.rs b/gix/src/status/index_worktree.rs index 2b72ddd2bd6..006bab8eeec 100644 --- a/gix/src/status/index_worktree.rs +++ b/gix/src/status/index_worktree.rs @@ -136,6 +136,15 @@ impl Repository { new_root: Some(workdir.to_owned()), }, )?; + let excludes_file = self + .config + .effective_excludes_file() + .map_err(crate::repository::attributes::Error::from)?; + let mut dirwalk_options: Option> = options.dirwalk_options.map(Into::into); + if let Some(options) = dirwalk_options.as_mut() { + options.use_untracked_cache = self.config.use_untracked_cache(); + options.untracked_cache_excludes_file = excludes_file.as_deref(); + } let out = gix_status::index_as_worktree_with_renames( index, @@ -165,7 +174,7 @@ impl Repository { fscache, }, fscache, - dirwalk: options.dirwalk_options.map(Into::into), + dirwalk: dirwalk_options, rewrites: options.rewrites, }, )?; diff --git a/gix/tests/gix/status.rs b/gix/tests/gix/status.rs index e2da4de21d8..7e80fd7bee2 100644 --- a/gix/tests/gix/status.rs +++ b/gix/tests/gix/status.rs @@ -16,6 +16,60 @@ pub fn repo(name: &str) -> crate::Result { )?) } +#[test] +fn untracked_cache_is_used_unless_disabled_by_config() -> crate::Result { + let tmp = gix_testtools::tempfile::tempdir()?; + let root = tmp.path(); + gix_testtools::git(root, "init")?; + for directory in 0..16 { + let directory = root.join(format!("d{directory}")); + std::fs::create_dir(&directory)?; + std::fs::write(directory.join("tracked"), b"content")?; + } + gix_testtools::git(root, "add .")?; + gix_testtools::git(root, "config core.untrackedCache true")?; + gix_testtools::git(root, "update-index --untracked-cache --index-version 2")?; + gix_testtools::git(root, "status --porcelain")?; + + let enabled = gix::ThreadSafeRepository::open_opts(root, crate::util::restricted())?.to_thread_local(); + let disabled = gix::ThreadSafeRepository::open_opts( + root, + crate::util::restricted().cli_overrides(["core.untrackedCache=false"]), + )? + .to_thread_local(); + let kept = gix::ThreadSafeRepository::open_opts( + root, + crate::util::restricted().cli_overrides(["core.untrackedCache=keep"]), + )? + .to_thread_local(); + + let read_dir_calls = |repo: &gix::Repository| -> crate::Result { + let mut status = repo.status(gix::progress::Discard)?.into_iter(None)?; + for item in status.by_ref() { + item?; + } + Ok(status + .into_outcome() + .expect("iteration was exhausted") + .index_worktree + .dirwalk + .expect("untracked files are enabled") + .read_dir_calls) + }; + let with_cache = read_dir_calls(&enabled)?; + let with_keep = read_dir_calls(&kept)?; + let without_cache = read_dir_calls(&disabled)?; + assert!( + with_cache < without_cache, + "core.untrackedCache=true should reduce directory reads, got {with_cache} with it and {without_cache} without" + ); + assert_eq!( + with_keep, with_cache, + "Git's `keep` value should retain an existing cache" + ); + Ok(()) +} + mod into_iter { use gix::status::{Item, Submodule, tree_index::TrackRenames}; use gix_diff::Rewrites;