Skip to content

Commit 2beeec9

Browse files
committed
modgraph: one walk per tree per process for glob expansion
A glob's walk is kept per root and start, and each pattern is matched against the kept listing, first by the literal text after its last star. Every directory the walk entered is examined again before the listing is reused, and a directory modified within two seconds of the walk is never trusted, so files that planning writes are seen. Before, a pattern with an empty literal prefix walked the whole package tree: the 127 source patterns of compat.libarchive, expanded about three times per plan, opened its 35 directories 13,406 times. Measured on a planned edit of one xlings source: 30,372 directory opens of installed packages fall to 1,749, the scan phase from 0.91 s to 43 ms, and ninja starts at 0.66 s instead of 3.05 s.
1 parent 0ef96b4 commit 2beeec9

2 files changed

Lines changed: 117 additions & 20 deletions

File tree

‎modules/manifest/src/glob.cppm‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,11 @@ void note_unnarrowable_path(const std::filesystem::path& p);
127127
// ("路径窄化不变式") and the user-facing behaviour in docs/04-mcpp-toml.md.
128128
std::vector<std::string> take_unnarrowable_paths();
129129

130+
// Does `relative`, a generic spelling relative to the glob's root, match
131+
// `glob`? The matching half of path_matches_glob, for a caller that already
132+
// holds the narrowed relative spelling (a cached directory listing).
133+
bool relative_path_matches_glob(std::string_view relative, std::string_view glob);
134+
130135
// Does `candidate` match `glob`, interpreted relative to `root`?
131136
//
132137
// Supports "**" (any number of directory levels) and "*" (within one segment).
@@ -149,7 +154,11 @@ bool path_matches_glob(const std::filesystem::path& candidate,
149154
note_unnarrowable_path(candidate);
150155
return false;
151156
}
157+
return relative_path_matches_glob(*rel, glob);
158+
}
152159

160+
bool relative_path_matches_glob(std::string_view relative, std::string_view glob)
161+
{
153162
auto match = [](std::string_view s, std::string_view p) -> bool {
154163
std::function<bool(std::size_t, std::size_t)> rec =
155164
[&](std::size_t si, std::size_t pi) -> bool {
@@ -185,7 +194,7 @@ bool path_matches_glob(const std::filesystem::path& candidate,
185194
};
186195
return rec(0, 0);
187196
};
188-
return match(*rel, glob);
197+
return match(relative, glob);
189198
}
190199

191200
} // namespace mcpp::modgraph

‎src/modgraph/scanner.cppm‎

Lines changed: 107 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -557,31 +557,56 @@ std::vector<std::string> expand_braces(std::string_view glob, int depthGuard) {
557557

558558
namespace {
559559

560-
// mcpp#225: the actual bounded recursive-directory walk for a SINGLE plain
561-
// glob (no `{` — expand_glob below desugars brace alternation via
562-
// expand_braces and calls this once per branch, unioning the results).
563-
std::vector<std::filesystem::path> expand_glob_one(const std::filesystem::path& root,
564-
std::string_view glob)
565-
{
566-
namespace fs = std::filesystem;
567-
std::vector<fs::path> out;
568-
if (!fs::exists(root)) return out;
560+
// A directory tree as a glob walk sees it, kept for the process.
561+
//
562+
// ONE WALK PER TREE (.agents/docs/2026-09-30-build-wall-time-progress-count-
563+
// and-hang-plan.md, F5 and W8). A package's `sources` are patterns, and a
564+
// pattern whose literal prefix is empty (`*/libarchive/archive_acl.c`) walked
565+
// the whole package tree. 127 such patterns, expanded about three times per
566+
// plan, opened libarchive's 35 directories 13,406 times, and 30,372 directory
567+
// opens of installed packages preceded the compile of every edited file. A
568+
// walk is now kept per root and start, and each pattern is matched against the
569+
// kept list.
570+
//
571+
// A KEPT WALK IS CHECKED, NOT TRUSTED. Planning writes files (a build
572+
// program's output, a descriptor's generated files), so every directory the
573+
// walk entered is examined again before the list is reused: adding, removing
574+
// or renaming an entry changes its directory's modification time. A directory
575+
// modified within two seconds of the walk is never trusted, since a coarse
576+
// clock could hide a change made in the same tick; a tree being edited is
577+
// therefore walked every time, as before.
578+
struct TreeListing {
579+
struct File {
580+
std::filesystem::path path;
581+
std::optional<std::string> relative; // to the glob root, generic and narrowed
582+
};
583+
std::vector<std::pair<std::filesystem::path, std::filesystem::file_time_type>> dirs;
584+
std::vector<File> files;
585+
bool trusted = true;
586+
};
569587

570-
// mcpp#225: bound the walk's start point to the glob's literal
571-
// directory prefix instead of always walking the whole root. A prefix
572-
// that doesn't exist means the glob can never match anything — return
573-
// empty WITHOUT walking (not a full-tree fallback).
574-
fs::path prefix = glob_literal_prefix(glob);
575-
fs::path start = prefix.empty() ? root : root / prefix;
576-
std::error_code startEc;
577-
if (!fs::exists(start, startEc)) return out;
588+
// mcpp#225: the bounded recursive-directory walk from `start`, with the
589+
// exclusions and the symlink-cycle guard every glob walk has.
590+
std::shared_ptr<const TreeListing> walk_tree(const std::filesystem::path& root,
591+
const std::filesystem::path& start) {
592+
namespace fs = std::filesystem;
593+
auto listing = std::make_shared<TreeListing>();
594+
const auto walkedAt = fs::file_time_type::clock::now();
595+
auto note_dir = [&](const fs::path& d) {
596+
std::error_code tec;
597+
const auto t = fs::last_write_time(d, tec);
598+
if (tec) { listing->trusted = false; return; }
599+
if (t > walkedAt - std::chrono::seconds(2)) listing->trusted = false;
600+
listing->dirs.emplace_back(d, t);
601+
};
602+
note_dir(start);
578603

579604
// Follow directory symlinks (vendored trees are often symlink farms).
580605
// Cycle guard: a directory whose canonical path is already on the
581606
// CURRENT recursion chain is a link loop — only that is pruned; the same
582607
// real directory reached via a second lexical path (dir + link to it)
583608
// still walks, because glob matching is lexical. Files reachable twice
584-
// are deduped by canonical identity afterwards.
609+
// are deduped by canonical identity by the caller.
585610
std::vector<fs::path> chain; // canonical dirs of the recursion stack
586611
std::error_code ec, eec; // ec: iteration; eec: per-entry probes
587612
{
@@ -604,15 +629,78 @@ std::vector<std::filesystem::path> expand_glob_one(const std::filesystem::path&
604629
it.disable_recursion_pending(); // link cycle
605630
} else {
606631
chain.push_back(eec ? e.path() : c);
632+
note_dir(e.path());
607633
}
608634
continue;
609635
}
610636
if (!e.is_regular_file(eec) || eec) continue;
611-
if (path_matches_glob(e.path(), root, glob)) out.push_back(e.path());
637+
auto rel = try_narrow(e.path().lexically_relative(root));
638+
// A name the code page cannot spell can never match a glob, and is
639+
// recorded as path_matches_glob records it.
640+
if (!rel) note_unnarrowable_path(e.path());
641+
listing->files.push_back({e.path(), std::move(rel)});
642+
}
643+
if (ec) listing->trusted = false;
644+
return listing;
645+
}
646+
647+
bool listing_current(const TreeListing& listing) {
648+
if (!listing.trusted) return false;
649+
std::error_code ec;
650+
for (auto const& [dir, time] : listing.dirs) {
651+
const auto now = std::filesystem::last_write_time(dir, ec);
652+
if (ec || now != time) return false;
653+
}
654+
return true;
655+
}
656+
657+
std::shared_ptr<const TreeListing> tree_listing(const std::filesystem::path& root,
658+
const std::filesystem::path& start) {
659+
static std::mutex m;
660+
static std::map<std::pair<std::filesystem::path, std::filesystem::path>,
661+
std::shared_ptr<const TreeListing>> kept;
662+
std::lock_guard lock(m);
663+
auto key = std::pair{root, start};
664+
if (auto it = kept.find(key); it != kept.end() && listing_current(*it->second))
665+
return it->second;
666+
auto listing = walk_tree(root, start);
667+
kept[key] = listing;
668+
return listing;
669+
}
670+
671+
// mcpp#225: the walk for a SINGLE plain glob (no `{` — expand_glob below
672+
// desugars brace alternation via expand_braces and calls this once per
673+
// branch, unioning the results), over the kept listing of its start.
674+
std::vector<std::filesystem::path> expand_glob_one(const std::filesystem::path& root,
675+
std::string_view glob)
676+
{
677+
namespace fs = std::filesystem;
678+
std::vector<fs::path> out;
679+
if (!fs::exists(root)) return out;
680+
681+
// mcpp#225: bound the walk's start point to the glob's literal
682+
// directory prefix instead of always walking the whole root. A prefix
683+
// that doesn't exist means the glob can never match anything — return
684+
// empty WITHOUT walking (not a full-tree fallback).
685+
fs::path prefix = glob_literal_prefix(glob);
686+
fs::path start = prefix.empty() ? root : root / prefix;
687+
std::error_code startEc;
688+
if (!fs::exists(start, startEc)) return out;
689+
690+
// Every match ends with the text after the glob's last `*`, which the
691+
// matcher reads literally: a cheap test that turns most entries away.
692+
const auto star = glob.find_last_of('*');
693+
const std::string_view tail = star == std::string_view::npos ? glob : glob.substr(star + 1);
694+
auto listing = tree_listing(root, start);
695+
for (auto const& f : listing->files) {
696+
if (!f.relative) continue;
697+
if (!tail.empty() && !f.relative->ends_with(tail)) continue;
698+
if (relative_path_matches_glob(*f.relative, glob)) out.push_back(f.path);
612699
}
613700
std::sort(out.begin(), out.end());
614701
// Dedup files reachable through more than one directory link (first
615702
// lexical occurrence wins).
703+
std::error_code eec;
616704
std::set<fs::path> seenFiles;
617705
out.erase(std::remove_if(out.begin(), out.end(), [&](const fs::path& p) {
618706
auto c = fs::canonical(p, eec);

0 commit comments

Comments
 (0)