Merge W3: bounded object addresses, a short tool scratch, extended-length paths (#641 item 3)

Author: speak-agent

docs/30-build-mcpp.md

@@ -1171,6 +1171,20 @@ left `mcpp run` printing the previous answer until the version was bumped.
 Entries accumulate as a tree is edited; `mcpp cache clean` empties the store,
 which lives at `<mcpp cache dir>/tool/<index>/<name>@<version>[+<source>]/`.
 
+**The paths a tool build writes are bounded (2026.9.15.2+).** The sub-build
+runs in `<mcpp cache dir>/tool/.build/<hash>/`, a directory named by the entry
+and the consuming project rather than nested inside the entry, and it is
+removed once the binary is published. A source that a build program selects
+from outside its own package, such as a file under a dependency's root named
+with `mcpp::source`, is compiled to
+`obj/<package>/__pkg/<owning package>/<path inside that package>`, or to
+`obj/<package>/__ext/<hash of its directory>/` when no package contains it, so
+its address does not grow with the distance between the two packages. On
+Windows, the engine's own build steps (`dyndep`, `stage`, the BMI edges,
+`coff-def` and check stamps) open their files through extended-length paths.
+Before these changes a tool's scan output reached 271 characters on a Windows
+runner and could not be read (#641).
+
 The action's own tracking is separate from the store's key. An action that
 declares the tool among its inputs re-runs when that file's bytes change,
 measured by overwriting the binary in the store: the artifact followed.

docs/zh/30-build-mcpp.md

@@ -984,6 +984,16 @@ grpc     = { version = "1.83.0", tools = ["grpc_cpp_plugin"] }
 版本被抬高。树被编辑时条目会累积;`mcpp cache clean` 清空 store,路径是
 `<mcpp cache dir>/tool/<index>/<name>@<version>[+<source>]/`。
 
+**工具构建写出的路径长度有界(2026.9.15.2+)。** 子构建在
+`<mcpp cache dir>/tool/.build/<hash>/` 中进行,这个目录由条目与消费工程命名,不嵌套在
+条目之内,二进制发布后即被删除。构建程序从自身包之外选入的源文件(例如用
+`mcpp::source` 选入依赖根下的文件)编译到
+`obj/<包>/__pkg/<所属包>/<在该包内的路径>`;没有任何包包含它时,编译到
+`obj/<包>/__ext/<其目录的哈希>/`。因此地址不随两个包之间的距离增长。在 Windows 上,
+引擎自己的构建步骤(`dyndep`、`stage`、BMI 相关的边、`coff-def` 与检查戳)以
+extended-length 路径打开文件。在这些改动之前,一个工具的扫描产物在 Windows runner
+上达到 271 个字符而无法读取(#641)。
+
 action 自己的跟踪与 store 的键是两回事。把工具列进 action 输入的规则,会在那个文件的
 字节变化时重跑 —— 实测直接覆盖 store 里的二进制,产物随之改变。
 

modules/buildmcpp/src/tool_store.cppm

@@ -122,6 +122,20 @@ std::string tree_stamp(const std::filesystem::path& root);
 
 // <cacheRoot>/tool/<index>/<pkg>@<ver>/<keyHex>/
 std::filesystem::path entry_dir(const std::filesystem::path& cacheRoot, const Key& k);
+
+// <cacheRoot>/tool/.build/<16 hex>/ -- where one consumer builds one entry.
+//
+// A SIBLING OF THE ENTRIES, NOT A CHILD OF ONE. The sub-build's own layout
+// (`target/<triple>/<fingerprint>/obj/...`) is appended to this path, and the
+// entry directory already spends its length on the index, the package, its
+// version with the source stamp and the key: 133 characters on a Windows
+// runner before the sub-build added its own (mcpp#641, item 3). The scratch
+// needs none of those names, only an identity per (entry, consumer) pair, so
+// that two projects building one tool at once do not share a ninja tree and a
+// re-run reuses its own.
+std::filesystem::path scratch_dir(const std::filesystem::path& cacheRoot,
+                                  const std::filesystem::path& entryDir,
+                                  const std::filesystem::path& consumerRoot);
 std::filesystem::path bin_path(const std::filesystem::path& entryDir,
                                std::string_view toolName,
                                std::string_view exeSuffix);
@@ -228,6 +242,18 @@ fs::path entry_dir(const fs::path& cacheRoot, const Key& k) {
          / key_hex(k);
 }
 
+fs::path scratch_dir(const fs::path& cacheRoot, const fs::path& entryDir,
+                     const fs::path& consumerRoot) {
+    // Identities, not narrow spellings: a home or project directory whose name
+    // the Windows code page cannot spell must not throw here.
+    const auto entry    = entryDir.lexically_normal().generic_u8string();
+    const auto consumer = consumerRoot.lexically_normal().generic_u8string();
+    std::string joined(reinterpret_cast<const char*>(entry.data()), entry.size());
+    joined += '\n';
+    joined.append(reinterpret_cast<const char*>(consumer.data()), consumer.size());
+    return cacheRoot / "tool" / ".build" / mcpp::toolchain::hash_string(joined);
+}
+
 fs::path bin_path(const fs::path& entryDir, std::string_view toolName,
                   std::string_view exeSuffix) {
     return entryDir / "bin" / (std::string(toolName) + std::string(exeSuffix));

modules/platform/src/fs.cppm

@@ -52,6 +52,30 @@ std::filesystem::path self_exe_path();
 //   POSIX:   `command -v <name>`
 std::optional<std::filesystem::path> which(std::string_view binary_name);
 
+// ── extended_length ───────────────────────────────────────────────────────
+//
+// The form of a path that a file operation may open whatever its length.
+//
+// A Win32 path that is not in the extended-length form is limited to 259
+// characters, and a relative path counts the working directory against that
+// limit. The engine subcommands ninja invokes (`mcpp dyndep`, `mcpp stage`,
+// the `bmi-*` edges, `coff-def`, the check stamps) receive paths relative to a
+// build directory, and in a host-tool sub-build those paths crossed the limit:
+// `clang-scan-deps` wrote a `.ddi` that `mcpp dyndep` then could not read
+// (mcpp#641, item 3).
+//
+//   Windows  the absolute, lexically normal path with backslashes, prefixed
+//            `\\?\` (a UNC path `\\server\share\x` becomes
+//            `\\?\UNC\server\share\x`); a path already in that form is
+//            returned unchanged.
+//   POSIX    the argument, unchanged. No limit of this kind exists.
+std::filesystem::path extended_length(const std::filesystem::path& p);
+
+// The Windows spelling rule of `extended_length`, as a pure function of an
+// absolute, generic (forward-slash or backslash) path string, so that the rule
+// is tested on every host. An empty or relative input is returned unchanged.
+std::string windows_extended_length_spelling(std::string_view absolutePath);
+
 // 不先删除目标文件，避免发布失败时丢失最后一份可用的编译数据库。
 bool replace_file(const std::filesystem::path& source,
                   const std::filesystem::path& destination,
@@ -141,6 +165,71 @@ std::optional<std::filesystem::path> which(std::string_view binary_name) {
     return std::filesystem::path(out);
 }
 
+std::string windows_extended_length_spelling(std::string_view in) {
+    std::string s(in);
+    for (auto& c : s) if (c == '/') c = '\\';
+    // Already extended-length (`\\?\`) or a device path (`\\.\`): the
+    // prefix disables every further interpretation, so nothing is rewritten.
+    if (s.starts_with("\\\\?\\") || s.starts_with("\\\\.\\")) return s;
+
+    std::string prefix;
+    std::string rest;
+    if (s.starts_with("\\\\")) {
+        prefix = "\\\\?\\UNC\\";
+        rest = s.substr(2);
+    } else if (s.size() >= 3 && std::isalpha(static_cast<unsigned char>(s[0]))
+               && s[1] == ':' && s[2] == '\\') {
+        prefix = "\\\\?\\";
+        rest = s;
+    } else {
+        return std::string(in);   // relative or rootless: not this function's input
+    }
+
+    // The prefix turns off `.`/`..` processing, so the components are resolved
+    // here, lexically. A `..` never climbs above the drive or the share.
+    std::vector<std::string> parts;
+    std::size_t keep = prefix.ends_with("UNC\\") ? 2 : 1;   // share or drive
+    std::size_t start = 0;
+    while (start <= rest.size()) {
+        auto end = rest.find('\\', start);
+        if (end == std::string::npos) end = rest.size();
+        std::string comp = rest.substr(start, end - start);
+        start = end + 1;
+        if (comp.empty() || comp == ".") continue;
+        if (comp == "..") {
+            if (parts.size() > keep) parts.pop_back();
+            continue;
+        }
+        parts.push_back(std::move(comp));
+    }
+    std::string out = prefix;
+    for (std::size_t i = 0; i < parts.size(); ++i) {
+        if (i) out += '\\';
+        out += parts[i];
+    }
+    // A bare drive keeps its root separator: `\\?\C:\`, not `\\?\C:`.
+    if (parts.size() == 1 && keep == 1) out += '\\';
+    return out;
+}
+
+std::filesystem::path extended_length(const std::filesystem::path& p) {
+#if defined(_WIN32)
+    if (p.empty()) return p;
+    std::error_code ec;
+    auto abs = std::filesystem::absolute(p, ec);
+    if (ec) return p;
+    // Through UTF-8, so that a name the ANSI code page cannot spell survives:
+    // the path narrowing rule (mcpp-contributing) forbids `.string()` here.
+    const auto u8 = abs.generic_u8string();
+    std::string narrow(reinterpret_cast<const char*>(u8.data()), u8.size());
+    const auto spelled = windows_extended_length_spelling(narrow);
+    return std::filesystem::path(std::u8string(
+        reinterpret_cast<const char8_t*>(spelled.data()), spelled.size()));
+#else
+    return p;
+#endif
+}
+
 bool replace_file(const std::filesystem::path& source,
                   const std::filesystem::path& destination,
                   std::error_code& ec) {

src/build/plan.cppm

@@ -1387,7 +1387,12 @@ make_plan(const mcpp::manifest::Manifest&         manifest,
     // match would file the member's sources under the outer package. Index 0 is
     // the root project; `packages.size()` means "outside every known root",
     // which is treated as root-owned and never cached.
-    auto owner_of = [&](const std::filesystem::path& src) -> std::size_t {
+    //
+    // `container_of` keeps the distinction `owner_of` folds away: no package
+    // contains the source at all. The object address of a source outside its
+    // declaring package needs it (see `outside_prefix` below).
+    auto container_of = [&](const std::filesystem::path& src)
+        -> std::optional<std::size_t> {
         std::size_t best = 0;
         std::size_t bestLen = 0;
         bool found = false;
@@ -1399,7 +1404,11 @@ make_plan(const mcpp::manifest::Manifest&         manifest,
             auto len = packages[p].root.generic_string().size();
             if (!found || len > bestLen) { best = p; bestLen = len; found = true; }
         }
-        return found ? best : 0;
+        if (!found) return std::nullopt;
+        return best;
+    };
+    auto owner_of = [&](const std::filesystem::path& src) -> std::size_t {
+        return container_of(src).value_or(0);
     };
 
     std::set<std::filesystem::path> scannedSources;
@@ -1469,6 +1478,47 @@ make_plan(const mcpp::manifest::Manifest&         manifest,
         return pkg.empty() ? safe
                            : std::filesystem::path(sanitize(pkg)) / safe;
     };
+    // A SOURCE OUTSIDE ITS DECLARING PACKAGE IS ADDRESSED BY WHERE IT IS, NOT
+    // BY HOW FAR IT IS FROM THAT PACKAGE (mcpp#641, item 3).
+    //
+    // `relPath` is relative to the package whose manifest or build program
+    // named the source. When a build program names a file under another
+    // package's root, mirroring that relPath spells one `__up` per directory
+    // between the two roots, so the address grew with a distance that has
+    // nothing to do with the file. Reported on windows-2022: a host tool's
+    // `.ddi` reached 271 characters, 90 of them this mirror, and `mcpp dyndep`
+    // could not open it.
+    //
+    // The address is instead the owning package's slug under a `__pkg`
+    // component, followed by the path inside that package; a source that no
+    // package contains is filed under `__ext/<hash of its directory>`. Both
+    // are downward and shell-safe by construction, and the marker components
+    // keep them apart from the declaring package's own mirrored directories.
+    // A source inside its declaring package does not reach this function, so
+    // its address is unchanged.
+    auto escapes_declaring_package = [](const std::filesystem::path& relPath) {
+        if (relPath.empty()) return false;
+        if (relPath.is_absolute() || relPath.has_root_name()) return true;
+        return *relPath.begin() == "..";
+    };
+    auto outside_prefix = [&](const std::filesystem::path& src)
+        -> std::filesystem::path {
+        if (auto c = container_of(src)) {
+            std::error_code ec;
+            auto rel = std::filesystem::relative(src, packages[*c].root, ec);
+            if (!ec && !rel.empty())
+                return std::filesystem::path("__pkg")
+                     / sanitize(qualified_package_name(packages[*c].manifest))
+                     / safe_object_prefix({}, rel.parent_path());
+        }
+        // The identity of a directory, not its spelling on this code page
+        // (see the path narrowing rule in mcpp-contributing).
+        const auto dir = src.parent_path().lexically_normal().generic_u8string();
+        const auto digest = mcpp::toolchain::hash_string(std::string_view(
+            reinterpret_cast<const char*>(dir.data()), dir.size()));
+        return std::filesystem::path("__ext") / digest.substr(0, 8);
+    };
+
     // mcpp#233/#240/#344: the single source of truth for a compile unit's
     // object addresses — scanned units AND the synthesized entry main go
     // through here, so neither the link input nor the cache address can
@@ -1483,12 +1533,20 @@ make_plan(const mcpp::manifest::Manifest&         manifest,
                           std::size_t owner) -> ObjectAddress
     {
         const auto fname = object_filename_for(src, objExt);
+        const bool escapes = escapes_declaring_package(relPath);
         if (owner == 0) {
             // Root project: never cached, historical layout preserved.
-            if (rootBasenameCount[fname] > 1)
+            if (rootBasenameCount[fname] > 1) {
+                if (escapes)
+                    return { std::filesystem::path("obj")
+                           / (pkg.empty() ? std::filesystem::path{}
+                                          : std::filesystem::path(sanitize(pkg)))
+                           / outside_prefix(src) / fname,
+                             {} };
                 return { std::filesystem::path("obj")
                        / safe_object_prefix(pkg, relPath.parent_path()) / fname,
                          {} };
+            }
             return { std::filesystem::path("obj") / fname, {} };
         }
 
@@ -1498,7 +1556,8 @@ make_plan(const mcpp::manifest::Manifest&         manifest,
         auto mirrored = safe_object_prefix({}, relPath.parent_path()) / fname;
 
         ObjectAddress addr;
-        addr.object = std::filesystem::path("obj") / slug / mirrored;
+        addr.object = std::filesystem::path("obj") / slug
+                    / (escapes ? outside_prefix(src) / fname : mirrored);
 
         // The cache address additionally has to be MACHINE-independent: another
         // machine computes the same key and reads the same entry. A relPath

src/build/prepare.cppm

@@ -9349,8 +9349,13 @@ prepare_build(bool print_fingerprint,
                     // Hashed rather than random so a re-run reuses its own
                     // scratch (ninja stays incremental if the publish step
                     // never got to delete it).
-                    sub.work_dir     = entry / std::format("build-{}",
-                        mcpp::toolchain::hash_string(workRoot.string()));
+                    //
+                    // Beside the entries rather than inside one: every
+                    // directory name of the entry is repeated in each object
+                    // path the sub-build writes, and on Windows those paths
+                    // crossed the 260-character limit (mcpp#641, item 3).
+                    sub.work_dir     = mcpp::build::tool_store::scratch_dir(
+                        cacheRoot, entry, workRoot);
                     sub.target_triple = "";            // HOST — the whole point
                     sub.profile       = "release";
                     sub.cache_mode    = overrides.cache_mode;

src/build/schedule/detach_codegen.cppm

@@ -123,6 +123,12 @@ struct CompileRequest {
     // itself reran.
     std::filesystem::path depFrom;
     std::filesystem::path depTo;
+    // The BMI as ninja spells it, which is what the copied depfile must name.
+    // `bmi` is the path this edge OPENS, and on Windows it is the absolute,
+    // extended-length form (mcpp::platform::fs::extended_length); ninja treats
+    // an edge whose depfile names anything but its own output spelling as
+    // permanently dirty. Empty means `bmi` is already that spelling.
+    std::string bmiTarget;
 };
 
 // Phase 1 — returns 0 as soon as the BMI is published, leaving code generation
@@ -372,7 +378,10 @@ int run_to_completion(std::string_view command,
     line += command;
 
     SECURITY_ATTRIBUTES sa{sizeof(sa), nullptr, TRUE};
-    HANDLE log = ::CreateFileA(logPath.string().c_str(), GENERIC_WRITE,
+    // The wide API: an extended-length log path (`bmi-supervise` receives one,
+    // see mcpp::platform::fs::extended_length) is honoured only by it, and a
+    // narrowed spelling would also throw on a name the code page cannot spell.
+    HANDLE log = ::CreateFileW(logPath.c_str(), GENERIC_WRITE,
                                FILE_SHARE_READ, &sa, CREATE_ALWAYS,
                                FILE_ATTRIBUTE_NORMAL, nullptr);
     // The command, in the log, before it runs. Phase 2 replays this file, so a
@@ -523,7 +532,8 @@ int compile_release_at_bmi(const CompileRequest& req) {
         // For a unit with no previous BMI that reduces to "it now exists".
         if (!req.bmi.empty() && bmi_identity(req.bmi) != before) {
             settle_bmi(req.bmi);
-            copy_first_rule(req.depFrom, req.depTo, req.bmi.string());
+            copy_first_rule(req.depFrom, req.depTo,
+                            req.bmiTarget.empty() ? req.bmi.string() : req.bmiTarget);
             return 0;                       // importers may proceed
         }
         if (const auto rc = read_rc(req.slot)) {
@@ -553,7 +563,8 @@ int compile_release_at_bmi(const CompileRequest& req) {
                 // in an importer — reproducible at `-j1`, so it was never a
                 // race between compilers, only between this loop's two checks.
                 settle_bmi(req.bmi);
-                copy_first_rule(req.depFrom, req.depTo, req.bmi.string());
+                copy_first_rule(req.depFrom, req.depTo,
+                                req.bmiTarget.empty() ? req.bmi.string() : req.bmiTarget);
             }
             return *rc;
         }

src/cli.cppm

@@ -32,6 +32,7 @@ import mcpp.wire;
 import mcpp.cli.cmd_sbom;
 import mcpp.platform.env;            // --offline → MCPP_OFFLINE
 import mcpp.platform.process;        // __action-stamp runs the checked command
+import mcpp.platform.fs;             // __action-stamp writes its stamps
 import mcpp.platform.runtime_search; // linker-wrapper path-injection opt-out
 import mcpp.ui;
 import mcpp.log;
@@ -1005,7 +1006,9 @@ int run(int argc, char** argv) {
         if (r != 0) return r;
         for (auto const& s : stamps) {
             std::error_code ec;
-            std::filesystem::path p{s};
+            // Relative to the build directory, which can be deep enough to
+            // take the stamp past the Windows path limit (mcpp#641, item 3).
+            const auto p = mcpp::platform::fs::extended_length(std::filesystem::path{s});
             if (!p.parent_path().empty())
                 std::filesystem::create_directories(p.parent_path(), ec);
             if (std::filesystem::exists(p, ec)) continue;