diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7592ddac..b90642da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,6 +58,7 @@ jobs: run: | python3 -m unittest scripts/test_verify_real_repo_lane_scale.py -v python3 -m unittest scripts/test_check_real_repo_lane_scale.py -v + python3 -m unittest scripts/test_check_real_framework_handoff.py -v - name: Check real-repository harness syntax run: bash -n scripts/verify-real-repo-lane-scale.sh diff --git a/.github/workflows/layered-workspaces.yml b/.github/workflows/layered-workspaces.yml index 4684b5cf..f8b60704 100644 --- a/.github/workflows/layered-workspaces.yml +++ b/.github/workflows/layered-workspaces.yml @@ -13,6 +13,9 @@ on: - "scripts/verify-linux-command-recipe-sandbox.sh" - "scripts/verify-windows-command-recipe-sandbox.ps1" - "scripts/verify-macos-nfs-framework-layers.sh" + - "scripts/verify-real-framework-handoff.sh" + - "scripts/check-real-framework-handoff.py" + - "scripts/test_check_real_framework_handoff.py" - "scripts/verify-environment-adapter-plugin.sh" - "scripts/verify-artifact-adapter-conformance.sh" - "scripts/verify-artifact-real-tool-gates.sh" @@ -37,6 +40,11 @@ on: required: false default: false type: boolean + run_real_framework_handoffs: + description: "Qualify pinned Go, pnpm, npm, Python, and CMake repositories through A -> B -> C macOS NFS lanes" + required: false + default: false + type: boolean jobs: core: @@ -232,6 +240,63 @@ jobs: - uses: Swatinem/rust-cache@v2 - run: scripts/verify-macos-nfs-framework-layers.sh + real-framework-candidate: + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_real_framework_handoffs }} + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + key: real-framework-candidate + - name: Build candidate Trail once + run: cargo build -p trail --release --locked + env: + CARGO_TARGET_DIR: ${{ runner.temp }}/trail-real-framework-target + - uses: actions/upload-artifact@v4 + with: + name: trail-real-framework-candidate-${{ github.sha }} + path: ${{ runner.temp }}/trail-real-framework-target/release/trail + if-no-files-found: error + + real-framework-handoffs: + needs: real-framework-candidate + strategy: + fail-fast: false + matrix: + framework: [go, pnpm, npm, python, cmake] + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: trail-real-framework-candidate-${{ github.sha }} + path: ${{ runner.temp }}/trail-real-framework-candidate + - name: Restore candidate executable permission + run: chmod 0755 "${{ runner.temp }}/trail-real-framework-candidate/trail" + - if: ${{ matrix.framework == 'go' }} + uses: actions/setup-go@v5 + with: + go-version: "1.26.x" + - if: ${{ matrix.framework == 'pnpm' || matrix.framework == 'npm' }} + uses: actions/setup-node@v4 + with: + node-version: "22" + - if: ${{ matrix.framework == 'pnpm' }} + run: corepack enable && corepack prepare pnpm@10.14.0 --activate + - name: Qualify ${{ matrix.framework }} A -> B -> C handoff + run: scripts/verify-real-framework-handoff.sh "${{ matrix.framework }}" + env: + TRAIL_BIN: ${{ runner.temp }}/trail-real-framework-candidate/trail + TRAIL_FRAMEWORK_EVIDENCE_DIR: ${{ runner.temp }}/trail-real-framework-${{ matrix.framework }} + TRAIL_FRAMEWORK_WORK_ROOT: ${{ runner.temp }}/trail-real-framework-work-${{ matrix.framework }} + - uses: actions/upload-artifact@v4 + if: ${{ always() }} + with: + name: trail-real-framework-${{ matrix.framework }} + path: ${{ runner.temp }}/trail-real-framework-${{ matrix.framework }} + if-no-files-found: error + dokan-conformance: if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_dokan_conformance }} runs-on: windows-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 58a330de..4fa2c1fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ All notable changes to Trail are documented in this file. Trail follows ### Fixed +- Managed lane commands now derive fixed policy, resolved executable, cache, + and output bindings from each active environment adapter instead of injecting + Cargo/npm defaults globally. Go, pnpm/npm/Yarn/Bun, Python, and CMake commands + receive isolated framework-native caches and exact tool paths, while inactive + frameworks no longer leak variables into the command. +- Node dependency executables and CMake build trees now bind directly from the + lane's generated upper, avoiding metadata-heavy build/dependency traversal + through macOS NFS while preserving a mounted source path and lane-private + mutation. Python also exposes its path-correct interpreter through + `TRAIL_VENV_PYTHON`. +- macOS NFS lane mounts now retain attributes and negative lookups for up to 60 + seconds within a mounted execution. Same-client mutations still invalidate + cached entries and synchronous writes remain enabled, while unchanged Go and + Node source/dependency walks avoid repeated userspace NFS round trips. - Lane/root diff addition and deletion totals now come from the emitted text diff rather than stable-line identity churn, so statistics agree with the unified patch while line-identity inspection remains available separately. @@ -35,6 +49,11 @@ All notable changes to Trail are documented in this file. Trail follows clone/reflink, Cargo revalidates the seed and recompiles affected workspace code, and lockfile, manifest, toolchain, target, platform, or build-policy changes still force an unseeded construction. +- Built-in Node dependency layers produced by a lockfile-frozen, + script-disabled install now allow public private-key example literals in + ordinary documentation, source, and type declarations. Strict scanning still + rejects secret-bearing paths such as `.env`, credential, `.pem`, and `.key` + files; custom or script-enabled producers receive no exemption. - `trail env ... --path .` now selects the repository-root component instead of failing path normalization. - Changed-path daemon authority now follows workspace generation changes and @@ -45,6 +64,10 @@ All notable changes to Trail are documented in this file. Trail follows ### Added +- Added an opt-in macOS real-framework qualification matrix for pinned bbolt, + date-fns, uuid, httpx, and LevelDB revisions. Each row produces checksummed + Agent A → B → C evidence for source-only handoff, parent-generation + inheritance, framework checks, cache/layer reuse, and lane-private outputs. - Added deterministic 10k/100k/1M artifact and source scale matrices for 1/5/20 lanes, fail-closed JSON evidence validation, and compositional owning-host NFS/FUSE/Dokan qualification that distinguishes mounted backend diff --git a/README.md b/README.md index 446818b1..77d890f5 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,14 @@ seed and recompiles affected workspace code; Trail never treats the predecessor as the final artifact for the new source root. Manifest, lockfile, toolchain, target, platform, or build-policy changes remain hard compatibility misses. +Managed command bindings are adapter-owned rather than Cargo-specific. Active +Go components receive shared module/build caches and an exact `TRAIL_GO`; +Node components receive package-manager caches plus a direct private +`TRAIL_NODE_MODULES`; Python components receive a lane-private `VIRTUAL_ENV` +and `TRAIL_VENV_PYTHON`; and CMake components receive a direct lane-private +`TRAIL_CMAKE_BUILD_DIR`. Inactive frameworks inject no cache, tool, or output +variables into the command. + ```sh trail env discover fix-login trail env plan fix-login diff --git a/docs/design/environment-adapter-contract.md b/docs/design/environment-adapter-contract.md index b6f7f124..5ff1e226 100644 --- a/docs/design/environment-adapter-contract.md +++ b/docs/design/environment-adapter-contract.md @@ -21,8 +21,9 @@ generation activation. The first built-ins are: graph-aware multi-module adapter is available; - `trail/cmake-build@1`: provisions a `writable_private` build tree with no synthetic shared layer. Configure is deliberately deferred until execution inside the mounted - lane so absolute paths in `CMakeCache.txt` name the stable lane workdir rather than a - disposable staging directory; + lane so its source path names the stable lane workdir rather than disposable staging; + the writable build directory is bound directly from the generated upper so configure, + compilation, and tests do not route build-tree metadata through NFS/FUSE/Dokan; - `trail/python-venv@1`: recognizes `pyproject.toml` and the common uv, Poetry, PDM, Pipenv, and requirements lock/manifest files, provisions a layer-free lane-private `.venv`, and keys it by every present dependency file plus the resolved Python @@ -57,6 +58,14 @@ scope. Host cache storage paths are rewritten to logical cache names before hash moving `.trail` storage cannot change artifact correctness identity. This projection does not advertise or invoke plugin protocol v3 and grants no v3-only capability. +Ordinary managed commands receive bindings declared by each active built-in adapter: +fixed policy values, resolved absolute tools, cache namespace subpaths, and generated +outputs. Direct output bindings are limited to lane-private or private-seeded policies; +they never expose a shared immutable layer for mutation. Binding-name collisions fail +closed, and an inactive adapter contributes nothing. This keeps command execution +framework-neutral while allowing Cargo targets, Node dependency trees, and CMake build +trees to bypass metadata-heavy filesystem transports safely. + The mapping deliberately preserves adapter semantics: Go's module/build stores remain performance-only while its vendor output remains an immutable private seed; CMake's path-bound build tree remains writable-private with no layer; OCI images and services diff --git a/docs/design/guardrails-security-and-redaction.md b/docs/design/guardrails-security-and-redaction.md index ad425195..8bd5a957 100644 --- a/docs/design/guardrails-security-and-redaction.md +++ b/docs/design/guardrails-security-and-redaction.md @@ -57,6 +57,15 @@ rejects credential-like content, protected paths, escaping links, special files, unsafe modes/xattrs, case collisions, concurrent mutation, and declared entry/byte/depth limit violations before publishing an envelope. +One narrow content-scanning distinction applies to the built-in Node adapter. +When a dependency tree is produced from a frozen lockfile with lifecycle +scripts disabled and a cleared environment, public example literals in ordinary +documentation, source, and type declarations do not taint the tree merely +because they spell a private-key marker. Secret-bearing paths (`.env`, +credentials, `.pem`, `.key`, and their protected equivalents) remain rejected. +Custom adapters, script-enabled installs, private-output promotion, and every +other artifact producer retain strict scanning. + ```mermaid flowchart TB Input["User, agent, CLI, HTTP, or MCP request"] diff --git a/docs/design/layered-lane-workspaces.md b/docs/design/layered-lane-workspaces.md index ef4bdf10..dcb0865b 100644 --- a/docs/design/layered-lane-workspaces.md +++ b/docs/design/layered-lane-workspaces.md @@ -738,6 +738,19 @@ The compiler cache is the primary cross-branch reuse mechanism. Target seeds are an optimization and may be disabled if toolchain behavior or absolute-path inputs make them unreliable. +### Managed command bindings + +The workspace host does not inject Cargo or npm variables unconditionally. +Each active built-in adapter declares its fixed policy values, resolved tools, +cache namespace paths, and output paths. Go receives `GOMODCACHE`, `GOCACHE`, +and `TRAIL_GO`; Node receives manager caches and a direct lane-private +`TRAIL_NODE_MODULES`; Python receives its mounted `VIRTUAL_ENV` and +`TRAIL_VENV_PYTHON`; CMake receives a direct writable-private +`TRAIL_CMAKE_BUILD_DIR`. Cargo retains direct private `CARGO_TARGET_DIR` and +managed `CARGO_HOME`/compiler-cache bindings. Output bindings may bypass the +mounted transport only for private-seeded, writable-private, or disposable +state, never for a live shared immutable directory. + ### Generic adapters A generic cache profile can declare: diff --git a/docs/guides/performance-and-scale-benchmarks.md b/docs/guides/performance-and-scale-benchmarks.md index fbc55113..14cafef7 100644 --- a/docs/guides/performance-and-scale-benchmarks.md +++ b/docs/guides/performance-and-scale-benchmarks.md @@ -118,6 +118,38 @@ TRAIL_SCALE_LABEL=manual-scale \ scripts/cli-scale-bench.sh ``` +## Real-Framework A → B → C Gate + +Framework adapters have a separate opt-in macOS qualification matrix. Each row +fetches an exact upstream revision, uses the candidate Trail binary, and moves +three native NFS lanes through a source edit, environment synchronization, and +real framework check. The gate covers bbolt/Go, date-fns/pnpm, uuid/npm, +httpx/Python virtual environments, and LevelDB/CMake: + +```sh +TRAIL_BIN=/absolute/path/to/trail \ +TRAIL_FRAMEWORK_EVIDENCE_DIR=/absolute/new/evidence/go \ +TRAIL_FRAMEWORK_WORK_ROOT=/absolute/new/work/go \ +scripts/verify-real-framework-handoff.sh go +``` + +Valid selectors are `go`, `pnpm`, `npm`, `python`, and `cmake`. The output +and optional work directories must not exist. When omitted, the work directory +defaults to `.work` and remains outside the uploaded evidence set. +`evidence.json` pins the repository/revision, three +distinct source roots, active component keys, layer identities, cache +namespaces, lane-private output identities, and SHA-256 digests of every raw +Trail report. The checker requires each edit to checkpoint only `README.md`, +each child to inherit its parent's active generation before editing, and every +framework command to pass. Go additionally requires compatible-predecessor +seeding with nonzero avoided bytes; npm/pnpm require one exact immutable layer; +Python and CMake require three distinct writable-private outputs. + +Dispatch `layered-workspaces.yml` with +`run_real_framework_handoffs=true` to run all five rows on clean macOS hosts +and upload their complete evidence directories. Synthetic tests or a skipped +matrix row are not production-readiness evidence for that framework. + ## Real-Repository Concurrent Lane Gate Use the blocking real-repository harness for release qualification of native-COW diff --git a/docs/lanes/large-repository-environments.md b/docs/lanes/large-repository-environments.md index 84238220..24d98f94 100644 --- a/docs/lanes/large-repository-environments.md +++ b/docs/lanes/large-repository-environments.md @@ -48,7 +48,7 @@ Use `immutable_shared` for read-only generated content, `writable_private` for persistent lane-only state, and `disposable` for scratch. Never model a live mutable tree as shared. -## Executable Cargo and Node examples +## Executable Cargo, Go, Node, Python, and CMake examples The built-in adapters need no framework-specific Trail configuration. In a Git-tracked Rust repository with `Cargo.toml` and `Cargo.lock`: @@ -85,9 +85,39 @@ trail lane exec node-b -- npm test ``` The immutable dependency lower is referenced by identity while consumer writes -go to each lane's private upper. Package-manager caches are performance-only +go to each lane's private upper. Trail exposes that upper directly through +`TRAIL_NODE_MODULES` and `NODE_PATH`, prepends its `.bin` directory to `PATH`, +and binds the selected package manager through `TRAIL_NPM`, `TRAIL_PNPM`, +`TRAIL_YARN`, or `TRAIL_BUN`. This avoids metadata-heavy dependency traversal +through the mounted source view. Package-manager caches are performance-only namespaces; they are never accepted as dependency correctness evidence. +The other built-ins use the same lane handoff: + +```sh +# Go: a shared module/build cache plus a lane-private vendor seed. +trail env sync all agent-a +trail lane exec agent-a -- sh -c '"$TRAIL_GO" test ./...' + +# Python: a lane-private, path-correct virtual environment. +trail env sync all agent-a +trail lane exec agent-a -- sh -c '"$TRAIL_VENV_PYTHON" -m compileall -q .' + +# CMake: configure and build outside the NFS/FUSE/Dokan transport. +trail env sync all agent-a +trail lane exec agent-a -- sh -c ' + "$TRAIL_CMAKE" -S . -B "$TRAIL_CMAKE_BUILD_DIR" + "$TRAIL_CMAKE" --build "$TRAIL_CMAKE_BUILD_DIR" --parallel +' +``` + +Go binds `GOMODCACHE`, `GOCACHE`, `TRAIL_GO`, and a local-toolchain/offline +vendor policy. Python binds `PIP_CACHE_DIR`, `UV_CACHE_DIR`, `VIRTUAL_ENV`, +`TRAIL_VENV_PYTHON`, and the venv executable directory. CMake binds the exact +host executable as `TRAIL_CMAKE` and a lane-private direct build path as +`TRAIL_CMAKE_BUILD_DIR`. A login shell may replace `PATH`; use the absolute +`TRAIL_*` executable variables in automated lane commands. + The framework-neutral TOML above is executable as `trail.environment.toml`. Create its declared input, record it, and run: diff --git a/scripts/check-real-framework-handoff.py b/scripts/check-real-framework-handoff.py new file mode 100755 index 00000000..742f8c65 --- /dev/null +++ b/scripts/check-real-framework-handoff.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Validate and seal one real-framework Agent A -> B -> C evidence directory.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +LANES = ("agent-a", "agent-b", "agent-c") + + +def load_report(raw: Path, name: str) -> dict[str, Any]: + value = json.loads((raw / f"{name}.json").read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise AssertionError(f"{name}.json is not a JSON object") + return value + + +def select_component(generation: dict[str, Any], component_id: str) -> dict[str, Any]: + matches = [ + item + for item in generation["components"] + if item["component_id"] == component_id + ] + if len(matches) != 1: + raise AssertionError( + f"expected one {component_id!r} component, found " + f"{[item['component_id'] for item in generation['components']]!r}" + ) + return matches[0] + + +def check_evidence( + evidence_dir: Path, + framework: str, + repository: str, + revision: str, + component_id: str, +) -> dict[str, Any]: + if framework not in {"go", "pnpm", "npm", "python", "cmake"}: + raise AssertionError(f"unsupported framework {framework!r}") + raw = evidence_dir / "raw" + generations = [load_report(raw, f"generation-{lane}") for lane in LANES] + syncs = [load_report(raw, f"sync-{lane}") for lane in LANES] + spawns = [load_report(raw, f"spawn-{lane}") for lane in LANES] + edits = [load_report(raw, f"edit-{lane}") for lane in LANES] + checks = [load_report(raw, f"check-{lane}") for lane in LANES] + components = [select_component(generation, component_id) for generation in generations] + workdirs = [spawn["workdir"] for spawn in spawns] + + if not all(generation["state"] == "active" for generation in generations): + raise AssertionError("all final generations must be active") + if len({generation["source_root"] for generation in generations}) != 3: + raise AssertionError("A, B, and C must have distinct source roots") + if len(set(workdirs)) != 3 or any(not Path(workdir).is_absolute() for workdir in workdirs): + raise AssertionError("A, B, and C must have distinct absolute lane workdirs") + if not all(check["exit_code"] == 0 for check in checks): + raise AssertionError("every framework check must exit successfully") + generated_dirty_paths = [] + for lane, edit in zip(LANES, edits, strict=True): + checkpoint = edit["lifecycle"]["checkpoint"] + if checkpoint["source_paths"] != ["README.md"]: + raise AssertionError(f"{lane} checkpointed unexpected source paths: {checkpoint!r}") + generated_dirty = checkpoint["generated_dirty_paths"] + if not isinstance(generated_dirty, int) or generated_dirty < 0: + raise AssertionError( + f"{lane} reported invalid generated-path accounting: {checkpoint!r}" + ) + generated_dirty_paths.append(generated_dirty) + + cache_namespaces = [ + {cache["name"]: cache["namespace_id"] for cache in item["caches"]} + for item in components + ] + if any(set(caches) != set(cache_namespaces[0]) for caches in cache_namespaces[1:]): + raise AssertionError("cache declarations differ across lanes") + for cache_name in cache_namespaces[0]: + if len({caches[cache_name] for caches in cache_namespaces}) != 1: + raise AssertionError(f"cache namespace {cache_name!r} was not inherited") + + component_keys = [item["component_key"] for item in components] + layer_ids = [item["layer_id"] for item in components] + output_storage = [ + {output["name"]: output["storage_identity"] for output in item["outputs"]} + for item in components + ] + if any(not storage for storage in output_storage): + raise AssertionError("each lane must report at least one output") + + if framework == "go": + if len(set(component_keys)) != 3: + raise AssertionError("Go source edits must produce exact distinct component keys") + if not all(layer_ids) or len(set(layer_ids)) != 3: + raise AssertionError("Go source edits must publish distinct exact vendor layers") + for lane, sync in zip(LANES[1:], syncs[1:], strict=True): + decisions = [ + decision + for decision in sync["decisions"] + if decision["component_id"] == component_id + ] + if len(decisions) != 1: + raise AssertionError(f"{lane} has unexpected decisions: {decisions!r}") + decision = decisions[0] + if decision["decision_source"] != "compatible_predecessor_seed": + raise AssertionError(f"{lane} did not seed from its predecessor: {decision!r}") + if not isinstance(decision["bytes_avoided"], int) or decision["bytes_avoided"] <= 0: + raise AssertionError(f"{lane} avoided no predecessor bytes: {decision!r}") + elif framework in {"pnpm", "npm"}: + if len(set(component_keys)) != 1: + raise AssertionError("Node dependency identity changed after source-only edits") + if not layer_ids[0] or len(set(layer_ids)) != 1: + raise AssertionError("Node lanes did not reuse one exact dependency layer") + else: + if len(set(component_keys)) != 1: + raise AssertionError("private environment identity changed after source-only edits") + if layer_ids != [None, None, None]: + raise AssertionError("Python/CMake private outputs must not publish shared layers") + if any( + not storage_identity.startswith("private_") + for storage in output_storage + for storage_identity in storage.values() + ): + raise AssertionError("Python/CMake outputs must use private storage contracts") + + shared_outputs = framework in {"go", "pnpm", "npm"} + for child_index, (child, parent_generation) in enumerate( + zip(LANES[1:], generations[:-1], strict=True), start=1 + ): + inheritance = spawns[child_index].get("environment_inheritance") + if shared_outputs: + if not isinstance(inheritance, dict) or inheritance.get("status") != "inherited": + raise AssertionError(f"{child} did not report inherited outputs: {inheritance!r}") + inherited = select_component( + load_report(raw, f"generation-before-edit-{child}"), component_id + ) + parent = select_component(parent_generation, component_id) + if inherited["component_key"] != parent["component_key"]: + raise AssertionError(f"{child} did not inherit its parent component key") + if inherited["layer_id"] != parent["layer_id"]: + raise AssertionError(f"{child} did not inherit its parent layer") + if inherited["caches"] != parent["caches"]: + raise AssertionError(f"{child} did not inherit its parent caches") + else: + if not isinstance(inheritance, dict): + raise AssertionError(f"{child} has no private-output decision report") + if ( + inheritance.get("status") != "skipped" + or inheritance.get("reason") != "no_compatible_outputs" + ): + raise AssertionError( + f"{child} unexpectedly inherited a lane-private output: {inheritance!r}" + ) + decisions = [ + item + for item in inheritance.get("outputs", []) + if item.get("component_id") == component_id + ] + if not decisions or any( + item.get("decision") != "private" + or item.get("reason") != "fresh_lane_private_upper" + for item in decisions + ): + raise AssertionError( + f"{child} did not require a fresh lane-private output: {decisions!r}" + ) + + raw_hashes = { + path.name: hashlib.sha256(path.read_bytes()).hexdigest() + for path in sorted(raw.glob("*.json")) + } + expected_names = { + "init.json", + *(f"spawn-{lane}.json" for lane in LANES), + *(f"edit-{lane}.json" for lane in LANES), + *(f"sync-{lane}.json" for lane in LANES), + *(f"check-{lane}.json" for lane in LANES), + *(f"generation-{lane}.json" for lane in LANES), + } + if shared_outputs: + expected_names.update( + { + "generation-before-edit-agent-b.json", + "generation-before-edit-agent-c.json", + } + ) + if set(raw_hashes) != expected_names: + raise AssertionError( + f"raw evidence set mismatch: missing={sorted(expected_names - set(raw_hashes))!r} " + f"extra={sorted(set(raw_hashes) - expected_names)!r}" + ) + + return { + "schema": "trail.real-framework-handoff/v1", + "framework": framework, + "repository": repository, + "revision": revision, + "backend": "nfs-cow", + "lanes": list(LANES), + "workdirs": workdirs, + "component_id": component_id, + "source_roots": [generation["source_root"] for generation in generations], + "component_keys": component_keys, + "layer_ids": layer_ids, + "cache_namespaces": cache_namespaces, + "output_storage": output_storage, + "generated_dirty_paths": generated_dirty_paths, + "assertions": { + "three_distinct_source_roots": True, + "shared_parent_generation_inherited_before_each_child_edit": shared_outputs, + "private_outputs_reprovisioned_per_child_lane": not shared_outputs, + "exactly_one_readme_source_path_per_edit": True, + "generated_paths_excluded_from_source_checkpoint": True, + "all_framework_checks_passed": True, + "framework_reuse_contract_passed": True, + }, + "raw_sha256": raw_hashes, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("evidence_dir", type=Path) + parser.add_argument("framework") + parser.add_argument("repository") + parser.add_argument("revision") + parser.add_argument("component_id") + args = parser.parse_args() + evidence = check_evidence( + args.evidence_dir, + args.framework, + args.repository, + args.revision, + args.component_id, + ) + encoded = json.dumps(evidence, indent=2, sort_keys=True) + "\n" + (args.evidence_dir / "evidence.json").write_text(encoded, encoding="utf-8") + print(encoded, end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_check_real_framework_handoff.py b/scripts/test_check_real_framework_handoff.py new file mode 100644 index 00000000..dfd7ed84 --- /dev/null +++ b/scripts/test_check_real_framework_handoff.py @@ -0,0 +1,186 @@ +import importlib.util +import json +import pathlib +import tempfile +import unittest + + +SCRIPT = pathlib.Path(__file__).with_name("check-real-framework-handoff.py") +SPEC = importlib.util.spec_from_file_location("real_framework_checker", SCRIPT) +assert SPEC and SPEC.loader +CHECKER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CHECKER) + + +class RealFrameworkHandoffCheckerTests(unittest.TestCase): + def write_report(self, raw, name, value): + (raw / f"{name}.json").write_text(json.dumps(value), encoding="utf-8") + + def fixture(self, root, framework): + raw = root / "raw" + raw.mkdir() + component_id = { + "go": "go-vendor", + "pnpm": "node", + "npm": "node", + "python": "python-venv", + "cmake": "cmake-build", + }[framework] + shared_key = "key-shared" + shared_layer = "layer-shared" + for index, lane in enumerate(CHECKER.LANES): + key = f"key-{index}" if framework == "go" else shared_key + if framework == "go": + layer = f"layer-{index}" + storage = layer + output_name = "vendor" + elif framework in {"pnpm", "npm"}: + layer = shared_layer + storage = layer + output_name = "node_modules" + else: + layer = None + storage = f"private_{index}" + output_name = "venv" if framework == "python" else "build-tree" + component = { + "component_id": component_id, + "component_key": key, + "layer_id": layer, + "caches": [ + { + "name": "cache", + "namespace_id": "cache-shared", + "protocol": "content_store", + } + ], + "outputs": [ + { + "name": output_name, + "storage_identity": storage, + } + ], + } + generation = { + "state": "active", + "source_root": f"root-{index}", + "components": [component], + } + self.write_report(raw, f"generation-{lane}", generation) + if index: + parent = json.loads( + (raw / f"generation-{CHECKER.LANES[index - 1]}.json").read_text( + encoding="utf-8" + ) + ) + if framework in {"go", "pnpm", "npm"}: + self.write_report(raw, f"generation-before-edit-{lane}", parent) + decision = { + "component_id": component_id, + "decision_source": ( + "compatible_predecessor_seed" if framework == "go" else "active_binding" + ), + "bytes_avoided": 100 if framework == "go" else 0, + } + self.write_report(raw, f"sync-{lane}", {"decisions": [decision]}) + self.write_report( + raw, + f"edit-{lane}", + { + "lifecycle": { + "checkpoint": { + "source_paths": ["README.md"], + "generated_dirty_paths": index + 1, + } + } + }, + ) + self.write_report(raw, f"check-{lane}", {"exit_code": 0}) + inheritance = None + if index: + if framework in {"go", "pnpm", "npm"}: + inheritance = { + "status": "inherited", + "reason": None, + "outputs": [], + } + else: + inheritance = { + "status": "skipped", + "reason": "no_compatible_outputs", + "outputs": [ + { + "component_id": component_id, + "decision": "private", + "reason": "fresh_lane_private_upper", + } + ], + } + self.write_report( + raw, + f"spawn-{lane}", + { + "lane": lane, + "workdir": f"/workspace/{lane}", + "environment_inheritance": inheritance, + }, + ) + self.write_report(raw, "init", {"initialized": True}) + return component_id + + def test_accepts_each_framework_contract(self): + for framework in ("go", "pnpm", "npm", "python", "cmake"): + with self.subTest(framework=framework), tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + component_id = self.fixture(root, framework) + evidence = CHECKER.check_evidence( + root, + framework, + "https://example.invalid/repository.git", + "a" * 40, + component_id, + ) + self.assertEqual(evidence["framework"], framework) + self.assertTrue(evidence["assertions"]["framework_reuse_contract_passed"]) + self.assertEqual(evidence["generated_dirty_paths"], [1, 2, 3]) + expected_raw = 18 if framework in {"go", "pnpm", "npm"} else 16 + self.assertEqual(len(evidence["raw_sha256"]), expected_raw) + + def test_rejects_an_edit_that_captures_more_than_the_readme(self): + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + component_id = self.fixture(root, "pnpm") + report = json.loads((root / "raw/edit-agent-b.json").read_text(encoding="utf-8")) + report["lifecycle"]["checkpoint"]["source_paths"].append("target/output") + self.write_report(root / "raw", "edit-agent-b", report) + with self.assertRaisesRegex(AssertionError, "unexpected source paths"): + CHECKER.check_evidence(root, "pnpm", "repo", "rev", component_id) + + def test_rejects_go_without_compatible_predecessor_seed(self): + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + component_id = self.fixture(root, "go") + report = json.loads((root / "raw/sync-agent-b.json").read_text(encoding="utf-8")) + report["decisions"][0]["decision_source"] = "singleflight_builder" + self.write_report(root / "raw", "sync-agent-b", report) + with self.assertRaisesRegex(AssertionError, "did not seed from its predecessor"): + CHECKER.check_evidence(root, "go", "repo", "rev", component_id) + + def test_rejects_private_output_inheritance(self): + with tempfile.TemporaryDirectory() as temp: + root = pathlib.Path(temp) + component_id = self.fixture(root, "python") + report = json.loads( + (root / "raw/spawn-agent-b.json").read_text(encoding="utf-8") + ) + report["environment_inheritance"] = { + "status": "inherited", + "reason": None, + "outputs": [], + } + self.write_report(root / "raw", "spawn-agent-b", report) + with self.assertRaisesRegex(AssertionError, "unexpectedly inherited"): + CHECKER.check_evidence(root, "python", "repo", "rev", component_id) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/verify-real-framework-handoff.sh b/scripts/verify-real-framework-handoff.sh new file mode 100755 index 00000000..ab99233c --- /dev/null +++ b/scripts/verify-real-framework-handoff.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Qualify one pinned real repository through an Agent A -> B -> C native-COW +# handoff. This is an opt-in release evidence gate, not a networked unit test. +set -euo pipefail +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) + +die() { + echo "verify-real-framework-handoff: $*" >&2 + exit 64 +} + +[[ $# == 1 ]] || die "usage: $0 " +framework=$1 +: "${TRAIL_BIN:?set TRAIL_BIN to the candidate Trail executable}" +: "${TRAIL_FRAMEWORK_EVIDENCE_DIR:?set TRAIL_FRAMEWORK_EVIDENCE_DIR to a new output directory}" +[[ $TRAIL_BIN == /* ]] || die "TRAIL_BIN must be absolute" +[[ -x $TRAIL_BIN ]] || die "TRAIL_BIN is not executable: $TRAIL_BIN" +[[ $TRAIL_FRAMEWORK_EVIDENCE_DIR == /* ]] || die "TRAIL_FRAMEWORK_EVIDENCE_DIR must be absolute" +[[ ! -e $TRAIL_FRAMEWORK_EVIDENCE_DIR ]] || die "evidence directory already exists" +qualification_root=${TRAIL_FRAMEWORK_WORK_ROOT:-$TRAIL_FRAMEWORK_EVIDENCE_DIR.work} +[[ $qualification_root == /* ]] || die "TRAIL_FRAMEWORK_WORK_ROOT must be absolute when set" +[[ ! -e $qualification_root ]] || die "qualification work directory already exists" + +case "$framework" in + go) + repository=https://github.com/etcd-io/bbolt.git + revision=55cb34b031c9855defb6c52db560a610f85bf5c3 + component_selector=go-vendor + component_id=go-vendor + ;; + pnpm) + repository=https://github.com/date-fns/date-fns.git + revision=4098115cf705e3af7f663d8e5b0686e39a9f478a + component_selector=node + component_id=node + ;; + npm) + repository=https://github.com/uuidjs/uuid.git + revision=b1da338815af4d919295eacb33aae340e372232a + component_selector=node + component_id=node + ;; + python) + repository=https://github.com/encode/httpx.git + revision=b5addb64f0161ff6bfe94c124ef76f6a1fba5254 + component_selector=python + component_id=python-venv + ;; + cmake) + repository=https://github.com/google/leveldb.git + revision=7ee830d02b623e8ffe0b95d59a74db1e58da04c5 + component_selector=cmake-build + component_id=cmake-build + ;; + *) die "unsupported framework: $framework" ;; +esac + +mkdir -p "$TRAIL_FRAMEWORK_EVIDENCE_DIR/raw" +repository_root=$qualification_root/repository +mkdir -p "$qualification_root" + +git -C "$qualification_root" init -q repository +git -C "$repository_root" remote add origin "$repository" +git -C "$repository_root" fetch -q --depth=1 origin "$revision" +git -C "$repository_root" checkout -q --detach FETCH_HEAD +[[ $(git -C "$repository_root" rev-parse HEAD) == "$revision" ]] || die "pinned revision mismatch" +run_json() { + local output=$1 + shift + local destination=$TRAIL_FRAMEWORK_EVIDENCE_DIR/raw/$output.json + local pending=$destination.pending + if ! "$TRAIL_BIN" --format json "$@" >"$pending"; then + rm -f -- "$pending" + return 1 + fi + mv -- "$pending" "$destination" +} + +run_edit() { + local lane=$1 + run_json "edit-$lane" lane exec "$lane" -- /bin/sh -c \ + 'printf "\nTrail real-framework qualification %s.\n" "$1" >> README.md' \ + trail "$lane" +} + +run_framework_check() { + local lane=$1 + case "$framework" in + go) + run_json "check-$lane" lane exec "$lane" -- /bin/sh -c \ + 'exec "$TRAIL_GO" test ./... -run "^TestTxStats_add$" 1>&2' + ;; + pnpm) + run_json "check-$lane" lane exec "$lane" -- /bin/sh -c \ + 'exec "$TRAIL_NODE" "$TRAIL_NODE_MODULES/oxfmt/bin/oxfmt" --check README.md 1>&2' + ;; + npm) + run_json "check-$lane" lane exec "$lane" -- /bin/sh -c \ + 'exec "$TRAIL_NODE" "$TRAIL_NODE_MODULES/typescript/bin/tsc" --version 1>&2' + ;; + python) + run_json "check-$lane" lane exec "$lane" -- /bin/sh -c \ + '"$TRAIL_VENV_PYTHON" -c '\''import os,sys; expected=os.path.join(os.getcwd(), ".venv"); assert os.path.realpath(sys.prefix) == os.path.realpath(os.environ["VIRTUAL_ENV"]) == os.path.realpath(expected)'\'' && exec "$TRAIL_VENV_PYTHON" -m compileall -q httpx 1>&2' + ;; + cmake) + run_json "check-$lane" lane exec "$lane" -- /bin/sh -c \ + 'set -eu + "$TRAIL_CMAKE" -S . -B "$TRAIL_CMAKE_BUILD_DIR" -DLEVELDB_BUILD_TESTS=OFF -DLEVELDB_BUILD_BENCHMARKS=OFF 1>&2 + exec "$TRAIL_CMAKE" --build "$TRAIL_CMAKE_BUILD_DIR" --target leveldb --parallel 2 1>&2' + ;; + esac +} + +cd "$repository_root" +run_json init init --from-git + +previous= +for lane in agent-a agent-b agent-c; do + if [[ -z $previous ]]; then + run_json "spawn-$lane" lane spawn "$lane" --from main --workdir-mode nfs-cow + else + run_json "spawn-$lane" lane spawn "$lane" --from "$previous" --workdir-mode nfs-cow + if [[ $framework == go || $framework == pnpm || $framework == npm ]]; then + run_json "generation-before-edit-$lane" env generation "$lane" + fi + fi + run_edit "$lane" + run_json "sync-$lane" env sync component "$component_id" \ + --adapter "$component_selector" --lane "$lane" + run_framework_check "$lane" + run_json "generation-$lane" env generation "$lane" + previous=$lane +done + +python3 "$SCRIPT_DIR/check-real-framework-handoff.py" \ + "$TRAIL_FRAMEWORK_EVIDENCE_DIR" "$framework" "$repository" "$revision" "$component_id" + +git diff --quiet -- || die "qualification mutated the Git checkout" +git diff --cached --quiet -- || die "qualification mutated the Git index" +echo "real-framework handoff evidence: $TRAIL_FRAMEWORK_EVIDENCE_DIR/evidence.json" diff --git a/trail/src/db/lane/managed_execution.rs b/trail/src/db/lane/managed_execution.rs index 89ebafbb..c568c4cd 100644 --- a/trail/src/db/lane/managed_execution.rs +++ b/trail/src/db/lane/managed_execution.rs @@ -263,9 +263,19 @@ impl Trail { .iter() .map(|node| (node.component_id.clone(), node.component_key.clone())) .collect::>(); - let has_environment = !desired_environment.is_empty() || !existing_environment.is_empty(); - let must_sync = has_environment - && !managed_environment_is_current(&desired_environment, &existing_environment); + // Legacy/manual layer bindings also have compatibility rows in + // workspace_environment_states, but they are not adapter-managed + // generations and must not trigger automatic framework discovery. + let had_active_generation = if view.is_some() { + self.active_environment_generation(lane)?.is_some() + } else { + false + }; + let must_sync = managed_environment_requires_sync( + &desired_environment, + &existing_environment, + had_active_generation, + ); if must_sync && view.is_none() { let error = Error::InvalidInput(format!( "lane `{lane}` declares workspace environments but does not use a layered COW workdir" @@ -284,7 +294,9 @@ impl Trail { return Err(error); } if must_sync { - if let Err(error) = self.sync_all_workspace_environments(lane, Some(&component_root)) { + let discovery_root = + (!desired_environment.is_empty()).then_some(component_root.as_str()); + if let Err(error) = self.sync_all_workspace_environments(lane, discovery_root) { self.push_managed_execution_phase( &mut phases, &branch.lane_id, @@ -841,13 +853,26 @@ impl Trail { } }; - drop(context.mount.take()); let had_mount = context.view.is_some(); + drop(context.mount.take()); + let unmount_error = if let Some(view) = context.view.as_ref() { + self.release_current_process_workspace_mount_lease(&view.view_id) + .err() + .map(|error| error.to_string()) + } else { + None + }; let _ = self.push_managed_context_phase( &mut context, "unmount", - if had_mount { "succeeded" } else { "skipped" }, - None, + if unmount_error.is_some() { + "failed" + } else if had_mount { + "succeeded" + } else { + "skipped" + }, + unmount_error.as_deref(), None, ); @@ -880,10 +905,17 @@ impl Trail { } else { "skipped" }; - let unmount_status = if had_mount { "succeeded" } else { "skipped" }; + let unmount_status = if unmount_error.is_some() { + "failed" + } else if had_mount { + "succeeded" + } else { + "skipped" + }; let errors = checkpoint_error .iter() .chain(disposal_error.iter()) + .chain(unmount_error.iter()) .cloned() .collect::>(); let finalization = ManagedExecutionFinalizationReceipt { @@ -1437,6 +1469,15 @@ fn managed_environment_is_current( }) } +fn managed_environment_requires_sync( + desired: &BTreeMap, + existing: &[WorkspaceEnvironmentReport], + had_active_generation: bool, +) -> bool { + (!desired.is_empty() || had_active_generation) + && !managed_environment_is_current(desired, existing) +} + fn managed_execution_output_pins( generation: &EnvironmentGenerationReport, bindings: &[ArtifactGenerationBindingReportV1], @@ -1724,9 +1765,30 @@ mod tests { )); assert!(!managed_environment_is_current( &BTreeMap::from([("python-venv".to_string(), "key-a".to_string())]), - &[ready] + std::slice::from_ref(&ready) )); assert!(managed_environment_is_current(&BTreeMap::new(), &[])); + + let manual = WorkspaceEnvironmentReport { + view_id: "view-a".to_string(), + adapter: "manual".to_string(), + expected_key: "manual-key".to_string(), + attached_key: Some("manual-key".to_string()), + status: "ready".to_string(), + reason: None, + updated_at: 1, + }; + assert!(!managed_environment_requires_sync( + &BTreeMap::new(), + &[manual], + false + )); + assert!(managed_environment_requires_sync( + &BTreeMap::new(), + std::slice::from_ref(&ready), + true + )); + assert!(managed_environment_requires_sync(&desired, &[], false)); } #[test] diff --git a/trail/src/db/lane/workdir/fuse.rs b/trail/src/db/lane/workdir/fuse.rs index dcf237d8..e9948757 100644 --- a/trail/src/db/lane/workdir/fuse.rs +++ b/trail/src/db/lane/workdir/fuse.rs @@ -1019,9 +1019,10 @@ mod fuse_overlay { .exec_lane_workspace( "rust-seed-b", &[ - "cargo".to_string(), - "build".to_string(), - "--offline".to_string(), + "sh".to_string(), + "-c".to_string(), + "cargo build --offline -vv --color never > target/trail-cargo.stdout 2> target/trail-cargo.stderr" + .to_string(), ], ) .unwrap(); @@ -1037,10 +1038,16 @@ mod fuse_overlay { .any(|layer_id| layer_id == layer.layer_id)); let view_b = db.lane_workspace_view("rust-seed-b").unwrap().unwrap(); let target_b = PathBuf::from(&view_b.generated_upper).join("target"); + let cargo_stderr = fs::read_to_string(target_b.join("trail-cargo.stderr")).unwrap(); + assert!( + cargo_stderr.contains("Fresh shared-dep"), + "the second lane did not reuse its private clone of the immutable target seed:\n{cargo_stderr}" + ); assert!( - !tree_has_name_fragment(&target_b, "libshared_dep"), - "the second lane rebuilt a dependency that was available in its immutable target seed" + !cargo_stderr.contains("Compiling shared-dep"), + "the second lane rebuilt a dependency that was available in its immutable target seed:\n{cargo_stderr}" ); + assert!(tree_has_name_fragment(&target_b, "libshared_dep")); assert!(!target_b.join("lane-a-private").exists()); assert!(tree_has_name_fragment( Path::new(&layer.storage_path), diff --git a/trail/src/db/lane/workdir/nfs_overlay.rs b/trail/src/db/lane/workdir/nfs_overlay.rs index 96a3acd9..fba4f2c1 100644 --- a/trail/src/db/lane/workdir/nfs_overlay.rs +++ b/trail/src/db/lane/workdir/nfs_overlay.rs @@ -468,19 +468,22 @@ mod macos { }; // Checkpointing reads the pinned source upper and authenticated view // journal directly, so it does not depend on an uncached READDIR of - // this mount. Retain a short attribute cache: disabling it makes every - // repeated parent lookup cross the userspace NFS server and turns - // framework builds with thousands of files into multi-minute metadata - // walks. Negative-name caching uses the same short lifetime. All live - // mutations pass through this one client mount, so CREATE/RENAME - // invalidates its own cached directory state; the cache only suppresses - // repeated misses from Node-style module resolution. macOS defaults to + // this mount. Each managed command receives a fresh mount, while all + // live mutations pass through the mount's one NFS client and invalidate + // that client's cached directory state. The managed-execution mount + // lifetime bounds this 60-second attribute/negative-name cache; + // explicit mount callers retain the same single-client mutation + // requirement. The cache suppresses repeated parent and source-file + // validation during Go, Cargo, Node, and CMake metadata walks. A + // one-second cache still made a verified Go build-cache hit spend more + // than a minute reopening an otherwise unchanged 158-file source tree. + // macOS defaults to // `nosync`, where a write syscall may return before this userspace server // has received the WRITE RPC. `sync` keeps successful writes as an // actual durability boundary; the server itself fsyncs every WRITE // before reporting FILE_SYNC. let opts = format!( - "locallocks,vers=3,tcp,sync,rsize=1048576,wsize=1048576,actimeo=1,nobrowse,port={port},mountport={port}" + "locallocks,vers=3,tcp,sync,rsize=1048576,wsize=1048576,actimeo=60,nobrowse,port={port},mountport={port}" ); let status = Command::new("/sbin/mount_nfs") .args(["-o", &opts, "127.0.0.1:/", &mountpoint.to_string_lossy()]) @@ -1400,7 +1403,7 @@ mod macos { .unwrap(); assert!(gate.success); assert_eq!(gate.source_root, checkpoint.root_id); - assert!(gate.environment_keys.is_empty()); + assert_eq!(gate.environment_keys, vec![layer.cache_key.clone()]); assert_eq!(gate.layer_ids, vec![layer.layer_id]); let newer_exec = db @@ -2197,9 +2200,10 @@ mod macos { .exec_lane_workspace( "rust-nfs-b", &[ - "cargo".to_string(), - "build".to_string(), - "--offline".to_string(), + "sh".to_string(), + "-c".to_string(), + "cargo build --offline -vv --color never > target/trail-cargo.stdout 2> target/trail-cargo.stderr" + .to_string(), ], ) .unwrap(); @@ -2215,10 +2219,16 @@ mod macos { .any(|layer_id| layer_id == layer.layer_id)); let view_b = db.lane_workspace_view("rust-nfs-b").unwrap().unwrap(); let target_b = PathBuf::from(&view_b.generated_upper).join("target"); + let cargo_stderr = fs::read_to_string(target_b.join("trail-cargo.stderr")).unwrap(); + assert!( + cargo_stderr.contains("Fresh shared-dep"), + "the second NFS lane did not reuse its private clone of the immutable target seed:\n{cargo_stderr}" + ); assert!( - !tree_has_name_fragment(&target_b, "libshared_dep"), - "the second NFS lane rebuilt a dependency present in its immutable target seed" + !cargo_stderr.contains("Compiling shared-dep"), + "the second NFS lane rebuilt a dependency present in its immutable target seed:\n{cargo_stderr}" ); + assert!(tree_has_name_fragment(&target_b, "libshared_dep")); assert!(!target_b.join("lane-a-private").exists()); assert!(tree_has_name_fragment( Path::new(&layer.storage_path), diff --git a/trail/src/db/lane/workspace_artifact.rs b/trail/src/db/lane/workspace_artifact.rs index 691c19b7..c0ee4fec 100644 --- a/trail/src/db/lane/workspace_artifact.rs +++ b/trail/src/db/lane/workspace_artifact.rs @@ -1583,7 +1583,7 @@ impl Trail { } fn ingest_artifact_file_bytes(&self, bytes: &[u8], mode: u32) -> Result { - self.ingest_artifact_file_bytes_with_path(bytes, mode, None) + self.ingest_artifact_file_bytes_with_path(bytes, mode, None, ArtifactSecretPolicy::Strict) } fn ingest_artifact_file_bytes_with_path( @@ -1591,13 +1591,14 @@ impl Trail { bytes: &[u8], mode: u32, relative_path: Option<&str>, + secret_policy: ArtifactSecretPolicy, ) -> Result { if mode & !0o777 != 0 { return Err(Error::InvalidInput(format!( "artifact file mode {mode:o} contains unsupported bits" ))); } - validate_artifact_secret_policy(bytes, relative_path)?; + validate_artifact_secret_policy(bytes, relative_path, secret_policy)?; let complete_hash = sha256_hex(bytes); let content = if bytes.len() <= ARTIFACT_WHOLE_BLOB_MAX_BYTES { let blob = ArtifactBlobV1 { @@ -1690,6 +1691,7 @@ impl Trail { path: &Path, relative_path: &str, mode: u32, + secret_policy: ArtifactSecretPolicy, ) -> Result { let before = fs::symlink_metadata(path)?; if !before.is_file() { @@ -1702,7 +1704,12 @@ impl Trail { let bytes = fs::read(path)?; let after = fs::symlink_metadata(path)?; ensure_artifact_file_unchanged(path, &before, &after, bytes.len() as u64)?; - return self.ingest_artifact_file_bytes_with_path(&bytes, mode, Some(relative_path)); + return self.ingest_artifact_file_bytes_with_path( + &bytes, + mode, + Some(relative_path), + secret_policy, + ); } let mut complete_hasher = Sha256::new(); @@ -1719,7 +1726,7 @@ impl Trail { path.display() )) })?; - validate_artifact_secret_policy(&boundary.data, Some(relative_path))?; + validate_artifact_secret_policy(&boundary.data, Some(relative_path), secret_policy)?; complete_hasher.update(&boundary.data); let chunk = ArtifactChunkV1 { version: ARTIFACT_CHUNK_VERSION, @@ -1793,6 +1800,17 @@ impl Trail { pub(crate) fn ingest_artifact_tree_under_write_lock( &self, source: &Path, + ) -> Result<(ArtifactTreeId, ArtifactTreeRootV1)> { + self.ingest_artifact_tree_under_write_lock_with_secret_policy( + source, + ArtifactSecretPolicy::Strict, + ) + } + + pub(crate) fn ingest_artifact_tree_under_write_lock_with_secret_policy( + &self, + source: &Path, + secret_policy: ArtifactSecretPolicy, ) -> Result<(ArtifactTreeId, ArtifactTreeRootV1)> { let root_before = fs::symlink_metadata(source)?; if root_before.file_type().is_symlink() || !root_before.is_dir() { @@ -1862,6 +1880,7 @@ impl Trail { entry.path(), &relative, normalized_artifact_file_mode(&metadata), + secret_policy, )?; directories .get_mut(parent) @@ -6445,7 +6464,17 @@ fn ensure_artifact_file_unchanged( Ok(()) } -fn validate_artifact_secret_policy(bytes: &[u8], relative_path: Option<&str>) -> Result<()> { +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ArtifactSecretPolicy { + Strict, + LockedPublicDependencies, +} + +fn validate_artifact_secret_policy( + bytes: &[u8], + relative_path: Option<&str>, + policy: ArtifactSecretPolicy, +) -> Result<()> { let Ok(text) = std::str::from_utf8(bytes) else { return Ok(()); }; @@ -6454,6 +6483,10 @@ fn validate_artifact_secret_policy(bytes: &[u8], relative_path: Option<&str>) -> upper.contains("-----BEGIN ") && upper.contains("PRIVATE KEY-----") }; let sensitive = match relative_path { + Some(path) if policy == ArtifactSecretPolicy::LockedPublicDependencies => { + is_secret_bearing_artifact_path(path) + && (contains_private_key || contains_sensitive_text(text)) + } Some(path) => { contains_private_key || is_secret_bearing_artifact_path(path) && contains_sensitive_text(text) @@ -7157,7 +7190,7 @@ mod tests { forward.source_closure.declared_inputs = inputs .iter() .map(|(name, value)| ArtifactResolutionInputV1 { - source_path: format!("inputs/{name}"), + source_path: format!("inputs/input-{name}"), content_hash: sha256_hex(&value.to_le_bytes()), size_bytes: 8, }) @@ -8138,6 +8171,21 @@ mod tests { let error = db.ingest_artifact_tree(source.path()).unwrap_err(); assert!(error.to_string().contains("private.pem")); assert!(error.to_string().contains("secret material")); + + let public_type_fixture = b"export type Example = '-----BEGIN PRIVATE KEY-----';\n"; + validate_artifact_secret_policy( + public_type_fixture, + Some("@example/openapi-types/types.d.ts"), + ArtifactSecretPolicy::LockedPublicDependencies, + ) + .unwrap(); + let error = validate_artifact_secret_policy( + public_type_fixture, + Some("package/private.key"), + ArtifactSecretPolicy::LockedPublicDependencies, + ) + .unwrap_err(); + assert!(error.to_string().contains("private.key")); } #[cfg(unix)] diff --git a/trail/src/db/lane/workspace_cargo.rs b/trail/src/db/lane/workspace_cargo.rs index 303e09fc..8f921e2d 100644 --- a/trail/src/db/lane/workspace_cargo.rs +++ b/trail/src/db/lane/workspace_cargo.rs @@ -1,10 +1,12 @@ use super::workspace_environment::{ resolve_workspace_tool_executable, WorkspaceEnvironmentAdapter, WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentAdapterProposal, - WorkspaceEnvironmentCacheAccess, WorkspaceEnvironmentCacheProtocol, - WorkspaceEnvironmentCommand, WorkspaceEnvironmentConstructionSeed, WorkspaceEnvironmentOutput, - WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, - WorkspaceEnvironmentResolutionInput, WorkspaceEnvironmentSandboxPolicy, + WorkspaceEnvironmentCacheAccess, WorkspaceEnvironmentCacheCommandBinding, + WorkspaceEnvironmentCacheProtocol, WorkspaceEnvironmentCommand, + WorkspaceEnvironmentConstructionSeed, WorkspaceEnvironmentOutput, + WorkspaceEnvironmentOutputCommandBinding, WorkspaceEnvironmentOutputPolicy, + WorkspaceEnvironmentPlan, WorkspaceEnvironmentResolutionInput, + WorkspaceEnvironmentSandboxPolicy, WorkspaceEnvironmentToolCommandBinding, }; use super::*; use crate::ids::sha256_hex; @@ -32,6 +34,46 @@ static CARGO_TARGET_SEED_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = "Locked Cargo target seed keyed by the complete source root and Rust toolchain identity", }; +const CARGO_CACHE_COMMAND_BINDINGS: &[WorkspaceEnvironmentCacheCommandBinding] = &[ + WorkspaceEnvironmentCacheCommandBinding { + cache_name: "cargo-home", + environment: "CARGO_HOME", + relative_path: "", + required: true, + }, + WorkspaceEnvironmentCacheCommandBinding { + cache_name: "sccache", + environment: "SCCACHE_DIR", + relative_path: "", + required: false, + }, +]; + +const CARGO_OUTPUT_COMMAND_BINDINGS: &[WorkspaceEnvironmentOutputCommandBinding] = + &[WorkspaceEnvironmentOutputCommandBinding { + output_name: "target-seed", + environment: Some("CARGO_TARGET_DIR"), + relative_path: "", + direct: true, + prepend_path: false, + required: true, + }]; + +const CARGO_TOOL_COMMAND_BINDINGS: &[WorkspaceEnvironmentToolCommandBinding] = &[ + WorkspaceEnvironmentToolCommandBinding { + programs: &["cargo"], + environment: "TRAIL_CARGO", + required: true, + prepend_path: true, + }, + WorkspaceEnvironmentToolCommandBinding { + programs: &["rustc"], + environment: "TRAIL_RUSTC", + required: true, + prepend_path: true, + }, +]; + #[cfg(target_os = "linux")] fn sccache_server_endpoint(cache_path: &Path) -> String { let digest = sha256_hex(cache_path.to_string_lossy().as_bytes()); @@ -69,6 +111,18 @@ impl WorkspaceEnvironmentAdapter for CargoTargetSeedAdapter { }) } + fn cache_command_bindings(&self) -> &'static [WorkspaceEnvironmentCacheCommandBinding] { + CARGO_CACHE_COMMAND_BINDINGS + } + + fn output_command_bindings(&self) -> &'static [WorkspaceEnvironmentOutputCommandBinding] { + CARGO_OUTPUT_COMMAND_BINDINGS + } + + fn tool_command_bindings(&self) -> &'static [WorkspaceEnvironmentToolCommandBinding] { + CARGO_TOOL_COMMAND_BINDINGS + } + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result { let root = normalize_component_root(component_root)?; Ok(db diff --git a/trail/src/db/lane/workspace_cmake.rs b/trail/src/db/lane/workspace_cmake.rs index 661dde40..ec8737f6 100644 --- a/trail/src/db/lane/workspace_cmake.rs +++ b/trail/src/db/lane/workspace_cmake.rs @@ -1,7 +1,9 @@ use super::workspace_environment::{ resolve_workspace_tool_executable, WorkspaceEnvironmentAdapter, WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentOutput, - WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, WorkspaceEnvironmentSandboxPolicy, + WorkspaceEnvironmentOutputCommandBinding, WorkspaceEnvironmentOutputPolicy, + WorkspaceEnvironmentPlan, WorkspaceEnvironmentSandboxPolicy, + WorkspaceEnvironmentToolCommandBinding, }; use super::*; @@ -16,7 +18,7 @@ static CMAKE_BUILD_TREE_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = name: "cmake-build", contract_major: 1, implementation_version: env!("CARGO_PKG_VERSION"), - distribution_digest: "builtin:cmake-build-plan-v1", + distribution_digest: "builtin:cmake-build-plan-v2", selectors: &["trail/cmake-build@1", "cmake-build", "cmake"], kind: "build", layer_adapter_name: "cmake-build", @@ -27,6 +29,24 @@ static CMAKE_BUILD_TREE_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = description: "Lane-private CMake build tree with configure deferred to the mounted lane", }; +const CMAKE_OUTPUT_COMMAND_BINDINGS: &[WorkspaceEnvironmentOutputCommandBinding] = + &[WorkspaceEnvironmentOutputCommandBinding { + output_name: "build-tree", + environment: Some("TRAIL_CMAKE_BUILD_DIR"), + relative_path: "", + direct: true, + prepend_path: false, + required: true, + }]; + +const CMAKE_TOOL_COMMAND_BINDINGS: &[WorkspaceEnvironmentToolCommandBinding] = + &[WorkspaceEnvironmentToolCommandBinding { + programs: &["cmake"], + environment: "TRAIL_CMAKE", + required: true, + prepend_path: true, + }]; + impl WorkspaceEnvironmentAdapter for CmakeBuildTreeAdapter { fn metadata(&self) -> &'static WorkspaceEnvironmentAdapterMetadata { &CMAKE_BUILD_TREE_ADAPTER_METADATA @@ -41,6 +61,14 @@ impl WorkspaceEnvironmentAdapter for CmakeBuildTreeAdapter { }) } + fn output_command_bindings(&self) -> &'static [WorkspaceEnvironmentOutputCommandBinding] { + CMAKE_OUTPUT_COMMAND_BINDINGS + } + + fn tool_command_bindings(&self) -> &'static [WorkspaceEnvironmentToolCommandBinding] { + CMAKE_TOOL_COMMAND_BINDINGS + } + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result { let root = normalize_component_root(component_root)?; Ok(db @@ -64,7 +92,7 @@ impl WorkspaceEnvironmentAdapter for CmakeBuildTreeAdapter { } let cmake = resolve_workspace_tool_executable("cmake")?; let implementation_version = env!("CARGO_PKG_VERSION").to_string(); - let distribution_digest = "builtin:cmake-build-plan-v1".to_string(); + let distribution_digest = "builtin:cmake-build-plan-v2".to_string(); let mount_path = join_repo_path(&component_root, "build"); let component_id = self.component_id(&component_root)?; let inputs = BTreeMap::from([ @@ -87,6 +115,11 @@ impl WorkspaceEnvironmentAdapter for CmakeBuildTreeAdapter { "configure_phase".to_string(), "deferred-to-mounted-lane".to_string(), ), + ( + "command_environment".to_string(), + "TRAIL_CMAKE_BUILD_DIR=direct-output:build-tree;TRAIL_CMAKE=tool:cmake;PATH+=tool-dir:cmake" + .to_string(), + ), ]); Ok(WorkspaceEnvironmentPlan { component_id, @@ -320,20 +353,35 @@ mod tests { #[cfg(target_os = "linux")] let mounted = db.mount_fuse_cow_workdir_for_lane(lane).unwrap(); let workdir = PathBuf::from(db.lane_workdir(lane).unwrap().workdir.unwrap()); + let environment = db + .lane_workspace_environment(lane) + .unwrap() + .into_iter() + .collect::>(); + let build_dir = PathBuf::from(&environment["TRAIL_CMAKE_BUILD_DIR"]); + let view = db.lane_workspace_view(lane).unwrap().unwrap(); + assert_eq!(build_dir, Path::new(&view.generated_upper).join("build")); + assert!(!build_dir.starts_with(&workdir)); let configured = Command::new("cmake") - .args(["-S", ".", "-B", "build", "-G", "Unix Makefiles"]) + .arg("-S") + .arg(".") + .arg("-B") + .arg(&build_dir) + .args(["-G", "Unix Makefiles"]) .current_dir(&workdir) .status() .unwrap(); assert!(configured.success()); let built = Command::new("cmake") - .args(["--build", "build", "--parallel", "2"]) + .arg("--build") + .arg(&build_dir) + .args(["--parallel", "2"]) .current_dir(&workdir) .status() .unwrap(); assert!(built.success()); - assert!(workdir.join("build/hello").is_file()); - let cache = fs::read_to_string(workdir.join("build/CMakeCache.txt")).unwrap(); + assert!(build_dir.join("hello").is_file()); + let cache = fs::read_to_string(build_dir.join("CMakeCache.txt")).unwrap(); assert!(cache.contains(workdir.to_string_lossy().as_ref())); drop(mounted); } @@ -343,21 +391,32 @@ mod tests { #[cfg(target_os = "linux")] let mounted = db.mount_fuse_cow_workdir_for_lane("cmake-a").unwrap(); let workdir_a = PathBuf::from(db.lane_workdir("cmake-a").unwrap().workdir.unwrap()); + let build_dir_a = db + .lane_workspace_environment("cmake-a") + .unwrap() + .into_iter() + .collect::>()["TRAIL_CMAKE_BUILD_DIR"] + .clone(); let cleaned = Command::new("cmake") - .args(["--build", "build", "--target", "clean"]) + .args(["--build", &build_dir_a, "--target", "clean"]) .current_dir(&workdir_a) .status() .unwrap(); assert!(cleaned.success()); - assert!(!workdir_a.join("build/hello").exists()); + assert!(!Path::new(&build_dir_a).join("hello").exists()); drop(mounted); #[cfg(target_os = "macos")] let mounted = db.mount_nfs_cow_workdir_for_lane("cmake-b").unwrap(); #[cfg(target_os = "linux")] let mounted = db.mount_fuse_cow_workdir_for_lane("cmake-b").unwrap(); - let workdir_b = PathBuf::from(db.lane_workdir("cmake-b").unwrap().workdir.unwrap()); - assert!(workdir_b.join("build/hello").is_file()); + let build_dir_b = db + .lane_workspace_environment("cmake-b") + .unwrap() + .into_iter() + .collect::>()["TRAIL_CMAKE_BUILD_DIR"] + .clone(); + assert!(Path::new(&build_dir_b).join("hello").is_file()); drop(mounted); assert!(db.list_workspace_layers().unwrap().is_empty()); } diff --git a/trail/src/db/lane/workspace_environment.rs b/trail/src/db/lane/workspace_environment.rs index bf399577..3a8e18da 100644 --- a/trail/src/db/lane/workspace_environment.rs +++ b/trail/src/db/lane/workspace_environment.rs @@ -134,6 +134,50 @@ pub(crate) struct WorkspaceEnvironmentCache { pub(crate) compatibility: BTreeMap, } +/// One adapter-declared binding from an active performance cache into normal +/// managed lane commands. Storage paths remain host-owned execution details; +/// adapters name only their logical cache and an optional contained subpath. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct WorkspaceEnvironmentCacheCommandBinding { + pub(crate) cache_name: &'static str, + pub(crate) environment: &'static str, + pub(crate) relative_path: &'static str, + pub(crate) required: bool, +} + +/// One adapter-declared fixed policy value applied to normal managed lane +/// commands while that adapter component is active. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct WorkspaceEnvironmentCommandBinding { + pub(crate) environment: &'static str, + pub(crate) value: &'static str, +} + +/// One adapter-declared executable binding for ordinary managed commands. +/// Resolution uses the same host policy as planning and exposes an absolute +/// executable path so login shells cannot silently select another toolchain. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct WorkspaceEnvironmentToolCommandBinding { + pub(crate) programs: &'static [&'static str], + pub(crate) environment: &'static str, + pub(crate) required: bool, + pub(crate) prepend_path: bool, +} + +/// One adapter-declared binding from an active generated output into normal +/// managed lane commands. Direct bindings bypass the filesystem transport for +/// metadata-heavy tool output while mounted bindings retain path-sensitive +/// prefixes such as Python virtual environments. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct WorkspaceEnvironmentOutputCommandBinding { + pub(crate) output_name: &'static str, + pub(crate) environment: Option<&'static str>, + pub(crate) relative_path: &'static str, + pub(crate) direct: bool, + pub(crate) prepend_path: bool, + pub(crate) required: bool, +} + /// Immutable identity owned by an external provider rather than Trail's /// filesystem layer store. Trail records and composes the identity but never /// deletes provider-owned content. @@ -588,6 +632,22 @@ pub(crate) trait WorkspaceEnvironmentAdapter: Sync { fn component_id(&self, component_root: &str) -> Result; + fn cache_command_bindings(&self) -> &'static [WorkspaceEnvironmentCacheCommandBinding] { + &[] + } + + fn command_bindings(&self) -> &'static [WorkspaceEnvironmentCommandBinding] { + &[] + } + + fn tool_command_bindings(&self) -> &'static [WorkspaceEnvironmentToolCommandBinding] { + &[] + } + + fn output_command_bindings(&self) -> &'static [WorkspaceEnvironmentOutputCommandBinding] { + &[] + } + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result; fn resolution_plan( @@ -639,7 +699,7 @@ pub(super) fn registered_environment_adapter_metadata( metadata } -fn builtin_environment_adapter_for_selector( +pub(super) fn builtin_environment_adapter_for_selector( selector: &str, ) -> Option<&'static dyn WorkspaceEnvironmentAdapter> { builtin_environment_adapters() diff --git a/trail/src/db/lane/workspace_go.rs b/trail/src/db/lane/workspace_go.rs index 358c69f4..74f364d6 100644 --- a/trail/src/db/lane/workspace_go.rs +++ b/trail/src/db/lane/workspace_go.rs @@ -1,8 +1,11 @@ use super::workspace_environment::{ resolve_workspace_tool_executable, WorkspaceEnvironmentAdapter, WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentCacheAccess, - WorkspaceEnvironmentCacheProtocol, WorkspaceEnvironmentCommand, WorkspaceEnvironmentOutput, + WorkspaceEnvironmentCacheCommandBinding, WorkspaceEnvironmentCacheProtocol, + WorkspaceEnvironmentCommand, WorkspaceEnvironmentCommandBinding, + WorkspaceEnvironmentConstructionSeed, WorkspaceEnvironmentOutput, WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, WorkspaceEnvironmentSandboxPolicy, + WorkspaceEnvironmentToolCommandBinding, }; use super::*; @@ -17,7 +20,7 @@ static GO_VENDOR_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = name: "go-vendor", contract_major: 1, implementation_version: env!("CARGO_PKG_VERSION"), - distribution_digest: "builtin:go-vendor-plan-v1", + distribution_digest: "builtin:go-vendor-plan-v2", selectors: &["trail/go-vendor@1", "go-vendor", "go"], kind: "dependency", layer_adapter_name: "go-vendor", @@ -28,6 +31,44 @@ static GO_VENDOR_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = description: "Single-module Go vendor tree with shared module and compiler caches", }; +const GO_CACHE_COMMAND_BINDINGS: &[WorkspaceEnvironmentCacheCommandBinding] = &[ + WorkspaceEnvironmentCacheCommandBinding { + cache_name: "module-store", + environment: "GOMODCACHE", + relative_path: "", + required: true, + }, + WorkspaceEnvironmentCacheCommandBinding { + cache_name: "build-cache", + environment: "GOCACHE", + relative_path: "", + required: true, + }, +]; + +const GO_COMMAND_BINDINGS: &[WorkspaceEnvironmentCommandBinding] = &[ + WorkspaceEnvironmentCommandBinding { + environment: "GOWORK", + value: "off", + }, + WorkspaceEnvironmentCommandBinding { + environment: "GOTOOLCHAIN", + value: "local", + }, + WorkspaceEnvironmentCommandBinding { + environment: "GOFLAGS", + value: "-mod=vendor -trimpath", + }, +]; + +const GO_TOOL_COMMAND_BINDINGS: &[WorkspaceEnvironmentToolCommandBinding] = + &[WorkspaceEnvironmentToolCommandBinding { + programs: &["go"], + environment: "TRAIL_GO", + required: true, + prepend_path: true, + }]; + impl WorkspaceEnvironmentAdapter for GoVendorAdapter { fn metadata(&self) -> &'static WorkspaceEnvironmentAdapterMetadata { &GO_VENDOR_ADAPTER_METADATA @@ -42,6 +83,18 @@ impl WorkspaceEnvironmentAdapter for GoVendorAdapter { }) } + fn cache_command_bindings(&self) -> &'static [WorkspaceEnvironmentCacheCommandBinding] { + GO_CACHE_COMMAND_BINDINGS + } + + fn command_bindings(&self) -> &'static [WorkspaceEnvironmentCommandBinding] { + GO_COMMAND_BINDINGS + } + + fn tool_command_bindings(&self) -> &'static [WorkspaceEnvironmentToolCommandBinding] { + GO_TOOL_COMMAND_BINDINGS + } + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result { let root = normalize_component_root(component_root)?; Ok(db @@ -79,7 +132,7 @@ impl WorkspaceEnvironmentAdapter for GoVendorAdapter { let go_version = command_identity("go", &["version"])?; let go_tool = resolve_workspace_tool_executable("go")?; let implementation_version = env!("CARGO_PKG_VERSION").to_string(); - let distribution_digest = "builtin:go-vendor-plan-v1".to_string(); + let distribution_digest = "builtin:go-vendor-plan-v2".to_string(); let cache_compatibility = BTreeMap::from([ ("go".to_string(), go_version.clone()), ("go_executable".to_string(), go_tool.identity.clone()), @@ -128,6 +181,11 @@ impl WorkspaceEnvironmentAdapter for GoVendorAdapter { "output_contract".to_string(), format!("immutable-seed-private:{mount_path}"), ), + ( + "command_environment".to_string(), + "GOMODCACHE=cache:module-store;GOCACHE=cache:build-cache;GOWORK=off;GOTOOLCHAIN=local;GOFLAGS=-mod=vendor -trimpath;TRAIL_GO=tool:go;PATH+=tool-dir:go" + .to_string(), + ), ]); inputs.insert( go_sum_path, @@ -172,7 +230,12 @@ impl WorkspaceEnvironmentAdapter for GoVendorAdapter { }, inputs: Vec::new(), resolution_inputs: Vec::new(), - construction_seed: None, + construction_seed: Some(WorkspaceEnvironmentConstructionSeed { + ignored_identity_inputs: BTreeSet::from([ + "source_root".to_string(), + "host:adapter_identity_v3".to_string(), + ]), + }), source_projection: Some((source_root.clone(), "project".to_string())), pre_commands: Vec::new(), command: Some(WorkspaceEnvironmentCommand { @@ -249,6 +312,7 @@ fn command_identity(program: &str, args: &[&str]) -> Result { #[cfg(test)] mod tests { use super::*; + use std::ffi::OsStr; #[test] fn go_adapter_vendors_once_and_reuses_the_immutable_tree_across_lanes() { @@ -367,6 +431,39 @@ mod tests { assert_eq!(status[0].component.component_id, "go-vendor"); assert_eq!(status[0].adapter.name, "go-vendor"); assert_eq!(status[0].status, "ready"); + let go_one_environment = db + .lane_workspace_environment("go-one") + .unwrap() + .into_iter() + .collect::>(); + let generation = db.active_environment_generation("go-one").unwrap().unwrap(); + let caches = generation.components[0] + .caches + .iter() + .map(|cache| (cache.name.as_str(), cache.namespace_id.as_str())) + .collect::>(); + assert_eq!( + Path::new(&go_one_environment["GOMODCACHE"]), + db.db_dir + .join("cache/namespaces") + .join(caches["module-store"]) + ); + assert_eq!( + Path::new(&go_one_environment["GOCACHE"]), + db.db_dir + .join("cache/namespaces") + .join(caches["build-cache"]) + ); + assert_eq!(go_one_environment["GOWORK"], "off"); + assert_eq!(go_one_environment["GOTOOLCHAIN"], "local"); + assert_eq!(go_one_environment["GOFLAGS"], "-mod=vendor -trimpath"); + assert!(Path::new(&go_one_environment["TRAIL_GO"]).is_absolute()); + assert_eq!( + std::env::split_paths(OsStr::new(&go_one_environment["PATH"])) + .next() + .unwrap(), + Path::new(&go_one_environment["TRAIL_GO"]).parent().unwrap() + ); let all = db.sync_all_workspace_environments("go-all", None).unwrap(); assert_eq!(all.generation.generation_sequence, 1); diff --git a/trail/src/db/lane/workspace_layer.rs b/trail/src/db/lane/workspace_layer.rs index 85816139..992800f3 100644 --- a/trail/src/db/lane/workspace_layer.rs +++ b/trail/src/db/lane/workspace_layer.rs @@ -2484,8 +2484,11 @@ impl Trail { let validated_entries = scan_layer_entries(&staging, false)?; sync_layer_tree(&staging)?; test_crash_point("layer_after_staging_sync"); - let (artifact_tree_id, artifact_tree) = - self.ingest_artifact_tree_under_write_lock(&staging)?; + let (artifact_tree_id, artifact_tree) = self + .ingest_artifact_tree_under_write_lock_with_secret_policy( + &staging, + workspace_layer_artifact_secret_policy(key), + )?; test_crash_point("layer_after_cas_tree"); let stable_entries = scan_layer_entries(&staging, false)?; if stable_entries != validated_entries { @@ -2698,8 +2701,11 @@ impl Trail { "workspace layer `{layer_id}` cannot recover because its published tree is corrupt" ))); } - let (artifact_tree_id, artifact_tree) = - self.ingest_artifact_tree_under_write_lock(final_path)?; + let (artifact_tree_id, artifact_tree) = self + .ingest_artifact_tree_under_write_lock_with_secret_policy( + final_path, + workspace_layer_artifact_secret_policy(key), + )?; verify_artifact_shadow_matches_layer_entries( &artifact_tree, &self.artifact_tree_flat_entries(&artifact_tree_id)?, @@ -6531,6 +6537,16 @@ pub(crate) fn scan_layer_entries( Ok(entries) } +fn workspace_layer_artifact_secret_policy( + key: &WorkspaceLayerKeyV1, +) -> super::workspace_artifact::ArtifactSecretPolicy { + if key.adapter == "node" && key.strategy.ends_with("-frozen-ignore-scripts-v1") { + super::workspace_artifact::ArtifactSecretPolicy::LockedPublicDependencies + } else { + super::workspace_artifact::ArtifactSecretPolicy::Strict + } +} + pub(crate) fn verify_artifact_shadow_matches_layer_entries( tree: &ArtifactTreeRootV1, artifact_entries: &BTreeMap, @@ -6771,6 +6787,28 @@ mod tests { } } + #[test] + fn only_script_disabled_node_layers_use_public_dependency_secret_policy() { + let mut node = key(); + node.strategy = "npm-frozen-ignore-scripts-v1".to_string(); + assert_eq!( + workspace_layer_artifact_secret_policy(&node), + super::super::workspace_artifact::ArtifactSecretPolicy::LockedPublicDependencies + ); + + node.strategy = "npm-frozen-allow-scripts-v1".to_string(); + assert_eq!( + workspace_layer_artifact_secret_policy(&node), + super::super::workspace_artifact::ArtifactSecretPolicy::Strict + ); + node.strategy = "npm-frozen-ignore-scripts-v1".to_string(); + node.adapter = "custom-node".to_string(); + assert_eq!( + workspace_layer_artifact_secret_policy(&node), + super::super::workspace_artifact::ArtifactSecretPolicy::Strict + ); + } + #[derive(Clone, Copy)] struct ArtifactConformanceProfile { producer_family: &'static str, @@ -7502,7 +7540,7 @@ validation = "path-contract" "INSERT INTO environment_component_states( view_id,component_id,adapter_identity,adapter_version,implementation_version, distribution_digest,kind,expected_key,attached_key,status,reason,updated_at) - VALUES('parent-view','node','trail/node@1',1,?1,'builtin:node-plan-v1', + VALUES('parent-view','node','trail/node@1',1,?1,'builtin:node-plan-v2', 'dependency',?2,?2,'ready',NULL,1)", params![env!("CARGO_PKG_VERSION"), &layer.cache_key], ) @@ -7558,7 +7596,7 @@ validation = "path-contract" db.conn .execute( "UPDATE environment_component_states - SET distribution_digest='builtin:node-plan-v1' + SET distribution_digest='builtin:node-plan-v2' WHERE view_id='parent-view' AND component_id='node'", [], ) diff --git a/trail/src/db/lane/workspace_node.rs b/trail/src/db/lane/workspace_node.rs index 689c5da2..2ecde3de 100644 --- a/trail/src/db/lane/workspace_node.rs +++ b/trail/src/db/lane/workspace_node.rs @@ -1,10 +1,12 @@ use super::workspace_environment::{ resolve_workspace_tool_executable, WorkspaceEnvironmentAdapter, WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentAdapterProposal, - WorkspaceEnvironmentCacheAccess, WorkspaceEnvironmentCacheProtocol, - WorkspaceEnvironmentCommand, WorkspaceEnvironmentInput, WorkspaceEnvironmentOutput, + WorkspaceEnvironmentCacheAccess, WorkspaceEnvironmentCacheCommandBinding, + WorkspaceEnvironmentCacheProtocol, WorkspaceEnvironmentCommand, WorkspaceEnvironmentInput, + WorkspaceEnvironmentOutput, WorkspaceEnvironmentOutputCommandBinding, WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, WorkspaceEnvironmentResolutionInput, WorkspaceEnvironmentSandboxPolicy, + WorkspaceEnvironmentToolCommandBinding, }; use super::*; use crate::ids::sha256_hex; @@ -20,7 +22,7 @@ static NODE_WORKSPACE_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = name: "node", contract_major: 1, implementation_version: env!("CARGO_PKG_VERSION"), - distribution_digest: "builtin:node-plan-v1", + distribution_digest: "builtin:node-plan-v2", selectors: &["trail/node@1", "node"], kind: "dependency", layer_adapter_name: "node", @@ -32,6 +34,99 @@ static NODE_WORKSPACE_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = "Frozen npm, pnpm, Yarn, or Bun dependency tree with a private writable lane upper", }; +const NODE_CACHE_COMMAND_BINDINGS: &[WorkspaceEnvironmentCacheCommandBinding] = &[ + WorkspaceEnvironmentCacheCommandBinding { + cache_name: "package-manager", + environment: "npm_config_cache", + relative_path: "npm", + required: true, + }, + WorkspaceEnvironmentCacheCommandBinding { + cache_name: "package-manager", + environment: "PNPM_HOME", + relative_path: "pnpm-home", + required: true, + }, + WorkspaceEnvironmentCacheCommandBinding { + cache_name: "package-manager", + environment: "PNPM_STORE_DIR", + relative_path: "pnpm-store", + required: true, + }, + WorkspaceEnvironmentCacheCommandBinding { + cache_name: "package-manager", + environment: "YARN_CACHE_FOLDER", + relative_path: "yarn", + required: true, + }, + WorkspaceEnvironmentCacheCommandBinding { + cache_name: "package-manager", + environment: "BUN_INSTALL_CACHE_DIR", + relative_path: "bun", + required: true, + }, +]; + +const NODE_TOOL_COMMAND_BINDINGS: &[WorkspaceEnvironmentToolCommandBinding] = &[ + WorkspaceEnvironmentToolCommandBinding { + programs: &["node"], + environment: "TRAIL_NODE", + required: true, + prepend_path: true, + }, + WorkspaceEnvironmentToolCommandBinding { + programs: &["npm"], + environment: "TRAIL_NPM", + required: false, + prepend_path: true, + }, + WorkspaceEnvironmentToolCommandBinding { + programs: &["pnpm"], + environment: "TRAIL_PNPM", + required: false, + prepend_path: true, + }, + WorkspaceEnvironmentToolCommandBinding { + programs: &["yarn"], + environment: "TRAIL_YARN", + required: false, + prepend_path: true, + }, + WorkspaceEnvironmentToolCommandBinding { + programs: &["bun"], + environment: "TRAIL_BUN", + required: false, + prepend_path: true, + }, +]; + +const NODE_OUTPUT_COMMAND_BINDINGS: &[WorkspaceEnvironmentOutputCommandBinding] = &[ + WorkspaceEnvironmentOutputCommandBinding { + output_name: "modules", + environment: Some("TRAIL_NODE_MODULES"), + relative_path: "", + direct: true, + prepend_path: false, + required: true, + }, + WorkspaceEnvironmentOutputCommandBinding { + output_name: "modules", + environment: Some("NODE_PATH"), + relative_path: "", + direct: true, + prepend_path: false, + required: true, + }, + WorkspaceEnvironmentOutputCommandBinding { + output_name: "modules", + environment: None, + relative_path: ".bin", + direct: true, + prepend_path: true, + required: true, + }, +]; + impl WorkspaceEnvironmentAdapter for NodeWorkspaceAdapter { fn metadata(&self) -> &'static WorkspaceEnvironmentAdapterMetadata { &NODE_WORKSPACE_ADAPTER_METADATA @@ -46,6 +141,18 @@ impl WorkspaceEnvironmentAdapter for NodeWorkspaceAdapter { }) } + fn cache_command_bindings(&self) -> &'static [WorkspaceEnvironmentCacheCommandBinding] { + NODE_CACHE_COMMAND_BINDINGS + } + + fn tool_command_bindings(&self) -> &'static [WorkspaceEnvironmentToolCommandBinding] { + NODE_TOOL_COMMAND_BINDINGS + } + + fn output_command_bindings(&self) -> &'static [WorkspaceEnvironmentOutputCommandBinding] { + NODE_OUTPUT_COMMAND_BINDINGS + } + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result { let root = normalize_package_root(component_root)?; Ok(db @@ -66,6 +173,9 @@ impl WorkspaceEnvironmentAdapter for NodeWorkspaceAdapter { { return Ok(None); } + if node_component_is_descendant_of_locked_workspace(db, source_root, &root)? { + return Ok(None); + } for (name, _) in supported_lockfiles() { if db .root_file_entry(source_root, &join_repo_path(&root, name))? @@ -319,18 +429,6 @@ impl Trail { display_package_root(&package_root) ))); } - if manager == "pnpm" - && self - .root_file_entry( - root_id, - &join_repo_path(&package_root, "pnpm-workspace.yaml"), - )? - .is_some() - { - return Err(Error::InvalidInput( - "pnpm workspace roots require the future monorepo environment adapter".to_string(), - )); - } if manager == "yarn" && (!manager_version.starts_with('1') || self @@ -359,7 +457,7 @@ impl Trail { } } let implementation_version = env!("CARGO_PKG_VERSION").to_string(); - let distribution_digest = "builtin:node-plan-v1".to_string(); + let distribution_digest = "builtin:node-plan-v2".to_string(); let mut key_inputs = files .iter() .map(|(path, entry)| (path.clone(), entry.content_hash.clone())) @@ -374,6 +472,10 @@ impl Trail { "adapter_distribution_digest".to_string(), distribution_digest.clone(), ); + key_inputs.insert( + "command_environment".to_string(), + "npm_config_cache=cache:package-manager/npm;PNPM_HOME=cache:package-manager/pnpm-home;PNPM_STORE_DIR=cache:package-manager/pnpm-store;YARN_CACHE_FOLDER=cache:package-manager/yarn;BUN_INSTALL_CACHE_DIR=cache:package-manager/bun;TRAIL_NODE=tool:node;TRAIL_NPM=tool?:npm;TRAIL_PNPM=tool?:pnpm;TRAIL_YARN=tool?:yarn;TRAIL_BUN=tool?:bun;TRAIL_NODE_MODULES=direct:node_modules;NODE_PATH=direct:node_modules;PATH+=direct:node_modules/.bin+tool-dirs".to_string(), + ); let key = WorkspaceLayerKeyV1 { kind: "dependency".to_string(), adapter: "node".to_string(), @@ -429,7 +531,12 @@ impl Trail { ]); let args = match manager.as_str() { "npm" => vec!["ci", "--ignore-scripts", "--no-audit", "--no-fund"], - "pnpm" => vec!["install", "--frozen-lockfile", "--ignore-scripts"], + "pnpm" => vec![ + "install", + "--ignore-workspace", + "--frozen-lockfile", + "--ignore-scripts", + ], "yarn" => vec!["install", "--frozen-lockfile", "--ignore-scripts"], "bun" => vec!["install", "--frozen-lockfile", "--ignore-scripts"], other => { @@ -509,6 +616,75 @@ impl Trail { } } +fn node_component_is_descendant_of_locked_workspace( + db: &Trail, + source_root: &ObjectId, + component_root: &str, +) -> Result { + if component_root.is_empty() { + return Ok(false); + } + for (lock_name, _) in supported_lockfiles() { + if db + .root_file_entry(source_root, &join_repo_path(component_root, lock_name))? + .is_some() + { + return Ok(false); + } + } + + let segments = component_root.split('/').collect::>(); + for depth in (0..segments.len()).rev() { + let ancestor = segments[..depth].join("/"); + let has_pnpm_workspace = db + .root_file_entry( + source_root, + &join_repo_path(&ancestor, "pnpm-workspace.yaml"), + )? + .is_some() + && db + .root_file_entry(source_root, &join_repo_path(&ancestor, "pnpm-lock.yaml"))? + .is_some(); + if has_pnpm_workspace { + return Ok(true); + } + + let package_path = join_repo_path(&ancestor, "package.json"); + let Some(package_entry) = db.root_file_entry(source_root, &package_path)? else { + continue; + }; + let package_bytes = db.materialize_entry_bytes(&package_entry)?; + let package: serde_json::Value = + serde_json::from_slice(&package_bytes).map_err(|error| { + Error::InvalidInput(format!( + "Node manifest `{package_path}` is malformed JSON: {error}" + )) + })?; + let declares_workspaces = + package + .get("workspaces") + .is_some_and(|workspaces| match workspaces { + serde_json::Value::Array(values) => !values.is_empty(), + serde_json::Value::Object(values) => values + .get("packages") + .and_then(serde_json::Value::as_array) + .is_some_and(|packages| !packages.is_empty()), + _ => false, + }); + if declares_workspaces { + for (lock_name, _) in supported_lockfiles() { + if db + .root_file_entry(source_root, &join_repo_path(&ancestor, lock_name))? + .is_some() + { + return Ok(true); + } + } + } + } + Ok(false) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct NodeResolutionSpec { manager: &'static str, @@ -779,6 +955,92 @@ mod tests { } } + #[test] + fn discovery_collapses_locked_workspace_descendants_but_keeps_nested_locks() { + let workspace = tempfile::tempdir().unwrap(); + fs::write( + workspace.path().join("package.json"), + r#"{"name":"root","private":true}"#, + ) + .unwrap(); + fs::write( + workspace.path().join("pnpm-lock.yaml"), + "lockfileVersion: '9.0'\n", + ) + .unwrap(); + fs::write( + workspace.path().join("pnpm-workspace.yaml"), + "packages:\n - packages/*\n", + ) + .unwrap(); + fs::create_dir_all(workspace.path().join("packages/member/example")).unwrap(); + fs::write( + workspace.path().join("packages/member/package.json"), + r#"{"name":"member"}"#, + ) + .unwrap(); + fs::write( + workspace + .path() + .join("packages/member/example/package.json"), + r#"{"name":"example"}"#, + ) + .unwrap(); + fs::create_dir_all(workspace.path().join("standalone")).unwrap(); + fs::write( + workspace.path().join("standalone/package.json"), + r#"{"name":"standalone"}"#, + ) + .unwrap(); + fs::write( + workspace.path().join("standalone/package-lock.json"), + r#"{"name":"standalone","lockfileVersion":3,"packages":{}}"#, + ) + .unwrap(); + + Trail::init(workspace.path(), "main", InitImportMode::WorkingTree, false).unwrap(); + let mut db = Trail::open(workspace.path()).unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "node-workspace", + Some("main"), + native_cow_mode(), + None, + None, + None, + &[], + false, + ) + .unwrap(); + let discovered = db + .discover_workspace_environment("node-workspace", None) + .unwrap(); + assert_eq!( + discovered + .components + .iter() + .filter(|component| component.adapter_identity == "trail/node@1") + .map(|component| component.component_id.as_str()) + .collect::>(), + ["node", "node:standalone"] + ); + if resolve_workspace_tool_executable("node").is_ok() + && resolve_workspace_tool_executable("pnpm").is_ok() + { + let plan = db + .plan_workspace_environment("node-workspace", "node", None) + .unwrap(); + assert_eq!( + plan.commands[0].args, + [ + "install", + "--ignore-workspace", + "--frozen-lockfile", + "--ignore-scripts" + ] + ); + } + } + #[test] fn package_manager_specific_snapshot_formats_are_distinct_and_validated() { let cases = [ @@ -1047,6 +1309,43 @@ mod tests { assert_eq!(cache_one.protocol, "content_store"); assert_eq!(cache_one.access, "tool_concurrent"); assert_eq!(cache_one.authority, "performance_only"); + let command_environment = db + .lane_workspace_environment("node-one") + .unwrap() + .into_iter() + .collect::>(); + let cache_root = db + .db_dir + .join("cache/namespaces") + .join(&cache_one.namespace_id); + for (name, relative) in [ + ("npm_config_cache", "npm"), + ("PNPM_HOME", "pnpm-home"), + ("PNPM_STORE_DIR", "pnpm-store"), + ("YARN_CACHE_FOLDER", "yarn"), + ("BUN_INSTALL_CACHE_DIR", "bun"), + ] { + assert_eq!( + Path::new(&command_environment[name]), + cache_root.join(relative) + ); + } + let direct_node_modules = Path::new(&view_one.generated_upper).join("node_modules"); + assert_eq!( + Path::new(&command_environment["TRAIL_NODE_MODULES"]), + direct_node_modules + ); + assert_eq!( + Path::new(&command_environment["NODE_PATH"]), + direct_node_modules + ); + assert_eq!( + std::env::split_paths(std::ffi::OsStr::new(&command_environment["PATH"])) + .next() + .unwrap(), + direct_node_modules.join(".bin") + ); + assert!(Path::new(&command_environment["TRAIL_NODE"]).is_absolute()); assert!(db .db_dir .join("cache/namespaces") diff --git a/trail/src/db/lane/workspace_python.rs b/trail/src/db/lane/workspace_python.rs index e3465f07..0c070ae1 100644 --- a/trail/src/db/lane/workspace_python.rs +++ b/trail/src/db/lane/workspace_python.rs @@ -2,9 +2,11 @@ use super::workspace_environment::{ resolve_workspace_tool_executable, workspace_mounted_commands_identity, WorkspaceEnvironmentAdapter, WorkspaceEnvironmentAdapterMetadata, WorkspaceEnvironmentAdapterProposal, WorkspaceEnvironmentCacheAccess, - WorkspaceEnvironmentCacheProtocol, WorkspaceEnvironmentCommand, WorkspaceEnvironmentInput, - WorkspaceEnvironmentOutput, WorkspaceEnvironmentOutputPolicy, WorkspaceEnvironmentPlan, - WorkspaceEnvironmentResolutionInput, WorkspaceEnvironmentSandboxPolicy, + WorkspaceEnvironmentCacheCommandBinding, WorkspaceEnvironmentCacheProtocol, + WorkspaceEnvironmentCommand, WorkspaceEnvironmentInput, WorkspaceEnvironmentOutput, + WorkspaceEnvironmentOutputCommandBinding, WorkspaceEnvironmentOutputPolicy, + WorkspaceEnvironmentPlan, WorkspaceEnvironmentResolutionInput, + WorkspaceEnvironmentSandboxPolicy, WorkspaceEnvironmentToolCommandBinding, }; use super::*; use crate::ids::sha256_hex; @@ -39,7 +41,7 @@ static PYTHON_VENV_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = name: "python-venv", contract_major: 1, implementation_version: env!("CARGO_PKG_VERSION"), - distribution_digest: "builtin:python-venv-plan-v2", + distribution_digest: "builtin:python-venv-plan-v3", selectors: &["trail/python-venv@1", "python-venv", "python"], kind: "dependency", layer_adapter_name: "python-venv", @@ -50,6 +52,65 @@ static PYTHON_VENV_ADAPTER_METADATA: WorkspaceEnvironmentAdapterMetadata = description: "Automatically initialized lane-private Python virtual environment at the stable mounted lane path", }; +const PYTHON_CACHE_COMMAND_BINDINGS: &[WorkspaceEnvironmentCacheCommandBinding] = &[ + WorkspaceEnvironmentCacheCommandBinding { + cache_name: "python-downloads", + environment: "PIP_CACHE_DIR", + relative_path: "pip", + required: true, + }, + WorkspaceEnvironmentCacheCommandBinding { + cache_name: "python-downloads", + environment: "UV_CACHE_DIR", + relative_path: "uv", + required: true, + }, +]; + +#[cfg(windows)] +const PYTHON_VENV_EXECUTABLE_DIRECTORY: &str = "Scripts"; +#[cfg(not(windows))] +const PYTHON_VENV_EXECUTABLE_DIRECTORY: &str = "bin"; +#[cfg(windows)] +const PYTHON_VENV_EXECUTABLE: &str = "Scripts/python.exe"; +#[cfg(not(windows))] +const PYTHON_VENV_EXECUTABLE: &str = "bin/python"; + +const PYTHON_OUTPUT_COMMAND_BINDINGS: &[WorkspaceEnvironmentOutputCommandBinding] = &[ + WorkspaceEnvironmentOutputCommandBinding { + output_name: "venv", + environment: Some("VIRTUAL_ENV"), + relative_path: "", + direct: false, + prepend_path: false, + required: true, + }, + WorkspaceEnvironmentOutputCommandBinding { + output_name: "venv", + environment: None, + relative_path: PYTHON_VENV_EXECUTABLE_DIRECTORY, + direct: false, + prepend_path: true, + required: true, + }, + WorkspaceEnvironmentOutputCommandBinding { + output_name: "venv", + environment: Some("TRAIL_VENV_PYTHON"), + relative_path: PYTHON_VENV_EXECUTABLE, + direct: false, + prepend_path: false, + required: true, + }, +]; + +const PYTHON_TOOL_COMMAND_BINDINGS: &[WorkspaceEnvironmentToolCommandBinding] = + &[WorkspaceEnvironmentToolCommandBinding { + programs: &["python3", "python"], + environment: "TRAIL_PYTHON", + required: true, + prepend_path: false, + }]; + impl WorkspaceEnvironmentAdapter for PythonVenvAdapter { fn metadata(&self) -> &'static WorkspaceEnvironmentAdapterMetadata { &PYTHON_VENV_ADAPTER_METADATA @@ -64,6 +125,18 @@ impl WorkspaceEnvironmentAdapter for PythonVenvAdapter { }) } + fn cache_command_bindings(&self) -> &'static [WorkspaceEnvironmentCacheCommandBinding] { + PYTHON_CACHE_COMMAND_BINDINGS + } + + fn output_command_bindings(&self) -> &'static [WorkspaceEnvironmentOutputCommandBinding] { + PYTHON_OUTPUT_COMMAND_BINDINGS + } + + fn tool_command_bindings(&self) -> &'static [WorkspaceEnvironmentToolCommandBinding] { + PYTHON_TOOL_COMMAND_BINDINGS + } + fn detect(&self, db: &Trail, source_root: &ObjectId, component_root: &str) -> Result { let root = normalize_python_component_root(component_root)?; for file in PYTHON_IDENTITY_FILES { @@ -176,7 +249,7 @@ impl WorkspaceEnvironmentAdapter for PythonVenvAdapter { let component_id = self.component_id(&component_root)?; let mount_path = join_python_path(&component_root, ".venv"); let implementation_version = env!("CARGO_PKG_VERSION").to_string(); - let distribution_digest = "builtin:python-venv-plan-v2".to_string(); + let distribution_digest = "builtin:python-venv-plan-v3".to_string(); let mut mounted_args = vec![ "-m".to_string(), "venv".to_string(), @@ -222,6 +295,12 @@ impl WorkspaceEnvironmentAdapter for PythonVenvAdapter { "mounted_action".to_string(), workspace_mounted_commands_identity(std::slice::from_ref(&mounted_command))?, ), + ( + "command_environment".to_string(), + format!( + "PIP_CACHE_DIR=cache:python-downloads/pip;UV_CACHE_DIR=cache:python-downloads/uv;VIRTUAL_ENV=output:venv;TRAIL_VENV_PYTHON=output:venv/{PYTHON_VENV_EXECUTABLE};PATH+=output:venv/{PYTHON_VENV_EXECUTABLE_DIRECTORY};TRAIL_PYTHON=tool:python3|python" + ), + ), ]); let source_resolution = python_source_resolution(db, source_root, &component_root)?; let managed_snapshot = if source_resolution.is_none() { @@ -1104,6 +1183,25 @@ mod tests { let mounted = db.mount_fuse_cow_workdir_for_lane(lane).unwrap(); let workdir = PathBuf::from(db.lane_workdir(lane).unwrap().workdir.unwrap()); assert!(workdir.join(".venv/pyvenv.cfg").is_file()); + let environment = db + .lane_workspace_environment(lane) + .unwrap() + .into_iter() + .collect::>(); + assert_eq!( + Path::new(&environment["VIRTUAL_ENV"]), + workdir.join(".venv") + ); + assert_eq!( + Path::new(&environment["TRAIL_VENV_PYTHON"]), + workdir.join(".venv/bin/python") + ); + assert_eq!( + std::env::split_paths(OsStr::new(&environment["PATH"])) + .next() + .unwrap(), + workdir.join(".venv/bin") + ); let venv_python = workdir.join(".venv/bin/python"); let prefix = Command::new(&venv_python) .args(["-c", "import sys; print(sys.prefix)"]) diff --git a/trail/src/db/lane/workspace_view.rs b/trail/src/db/lane/workspace_view.rs index 32639290..57a2f087 100644 --- a/trail/src/db/lane/workspace_view.rs +++ b/trail/src/db/lane/workspace_view.rs @@ -92,10 +92,16 @@ impl WorkspaceMountLease { |row| row.get::<_, String>(0), ) .optional()?; - db.conn.execute( + let updated = db.conn.execute( "UPDATE workspace_views SET status = 'unmounted', owner_pid = NULL, owner_start_token = NULL, heartbeat_at = NULL, updated_at = ?1 WHERE view_id = ?2 AND owner_start_token = ?3", params![now_ts(), self.view_id, self.owner_start_token], )?; + if updated != 1 { + return Err(Error::InvalidInput(format!( + "workspace view `{}` mount lease changed before release", + self.view_id + ))); + } if let Some(meta_dir) = view { let _ = fs::remove_file(Path::new(&meta_dir).join("mount.json")); } @@ -111,6 +117,52 @@ impl Drop for WorkspaceMountLease { } impl Trail { + pub(crate) fn release_current_process_workspace_mount_lease( + &self, + view_id: &str, + ) -> Result<()> { + let owner = self + .conn + .query_row( + "SELECT owner_pid, owner_start_token, meta_dir FROM workspace_views WHERE view_id = ?1", + params![view_id], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + .optional()?; + let Some((owner_pid, owner_token, meta_dir)) = owner else { + return Err(Error::Corrupt(format!( + "workspace view `{view_id}` disappeared during managed unmount" + ))); + }; + let Some((owner_pid, owner_token)) = owner_pid.zip(owner_token) else { + return Ok(()); + }; + let current_pid = std::process::id(); + let current_token = current_process_start_token(); + if owner_pid != current_pid || owner_token != current_token { + // The managed mount is gone and a different owner acquired the + // view after it. Never clear that successor's lease. + return Ok(()); + } + let updated = self.conn.execute( + "UPDATE workspace_views SET status = 'unmounted', owner_pid = NULL, owner_start_token = NULL, heartbeat_at = NULL, updated_at = ?1 WHERE view_id = ?2 AND owner_pid = ?3 AND owner_start_token = ?4", + params![now_ts(), view_id, current_pid, current_token], + )?; + if updated != 1 { + return Err(Error::InvalidInput(format!( + "workspace view `{view_id}` mount lease changed during managed unmount" + ))); + } + let _ = fs::remove_file(Path::new(&meta_dir).join("mount.json")); + Ok(()) + } + /// Start a mount worker owned by the current long-lived Trail process. /// HTTP/MCP daemons use this non-blocking entry point; the worker still /// owns the same foreground lifecycle and durable lease as the CLI path. @@ -933,32 +985,11 @@ impl Trail { source_root: &ObjectId, component_root: &str, ) -> Result> { - let node_cache = self.db_dir.join("cache/tool-home/node/npm"); let component_root = if component_root.is_empty() || component_root == "." { String::new() } else { normalize_relative_path(component_root)? }; - let cargo_component = if component_root.is_empty() { - "cargo-target-seed".to_string() - } else { - format!("cargo-target-seed:{component_root}") - }; - let cargo_home = self - .active_workspace_cache_path(view, &cargo_component, "cargo-home")? - .unwrap_or_else(|| self.db_dir.join("cache/tool-home/cargo")); - let sccache_dir = self - .active_workspace_cache_path(view, &cargo_component, "sccache")? - .unwrap_or_else(|| self.db_dir.join("cache/tool-home/sccache")); - let target_mount = if component_root.is_empty() { - "target".to_string() - } else { - format!("{component_root}/target") - }; - let target_dir = self.prepare_direct_private_seed(view, &target_mount)?; - for path in [&cargo_home, &node_cache, &sccache_dir] { - fs::create_dir_all(path)?; - } let mut environment = vec![ ( "TRAIL_WORKSPACE".to_string(), @@ -971,23 +1002,16 @@ impl Trail { "TRAIL_VIEW_GENERATION".to_string(), view.generation.to_string(), ), - ( - "CARGO_HOME".to_string(), - cargo_home.to_string_lossy().into_owned(), - ), - ( - "CARGO_TARGET_DIR".to_string(), - target_dir.to_string_lossy().into_owned(), - ), - ( - "SCCACHE_DIR".to_string(), - sccache_dir.to_string_lossy().into_owned(), - ), - ( - "npm_config_cache".to_string(), - node_cache.to_string_lossy().into_owned(), - ), ]; + environment.extend( + self.active_workspace_command_bindings(view, &component_root) + .map_err(|error| { + Error::InvalidInput(format!( + "workspace view `{}` command bindings could not be resolved: {error}", + view.view_id + )) + })?, + ); if let Some(generation_id) = self .conn .query_row( @@ -1060,15 +1084,27 @@ impl Trail { serde_json::to_string(&services)?, )); } - if command_available("sccache") { + let has_cargo_target = environment + .iter() + .any(|(name, _)| name == "CARGO_TARGET_DIR"); + let has_sccache_cache = environment.iter().any(|(name, _)| name == "SCCACHE_DIR"); + if has_sccache_cache && command_available("sccache") { environment.extend([ ("RUSTC_WRAPPER".to_string(), "sccache".to_string()), ("CARGO_INCREMENTAL".to_string(), "0".to_string()), ]); - } else { + } else if has_cargo_target { environment.push(("CARGO_INCREMENTAL".to_string(), "1".to_string())); } - if let Some(shadow) = self.ensure_workspace_git_shadow(view, source_root)? { + if let Some(shadow) = self + .ensure_workspace_git_shadow(view, source_root) + .map_err(|error| { + Error::InvalidInput(format!( + "workspace view `{}` Git shadow could not be prepared: {error}", + view.view_id + )) + })? + { environment.extend([ ("GIT_DIR".to_string(), shadow.git_dir.clone()), ("GIT_WORK_TREE".to_string(), shadow.work_tree.clone()), @@ -1085,27 +1121,272 @@ impl Trail { Ok(environment) } + fn active_workspace_command_bindings( + &self, + view: &LaneWorkspaceViewReport, + component_root: &str, + ) -> Result> { + let mut statement = self.conn.prepare( + "SELECT c.component_id, c.adapter_identity + FROM environment_view_generations active + JOIN environment_generation_components c + ON c.generation_id = active.generation_id + WHERE active.view_id = ?1 + ORDER BY c.component_id, c.adapter_identity", + )?; + let components = statement + .query_map(params![&view.view_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()?; + let mut bindings = BTreeMap::::new(); + let mut output_path_prefixes = Vec::::new(); + let mut tool_path_prefixes = Vec::::new(); + for (component_id, adapter_identity) in components { + let Some(adapter) = + super::workspace_environment::builtin_environment_adapter_for_selector( + &adapter_identity, + ) + else { + continue; + }; + if adapter.component_id(component_root)? != component_id { + continue; + } + for binding in adapter.command_bindings() { + insert_workspace_command_binding( + &mut bindings, + binding.environment, + binding.value.to_string(), + &component_id, + )?; + } + for binding in adapter.tool_command_bindings() { + let mut resolved = None; + let mut errors = Vec::new(); + for program in binding.programs { + match super::workspace_environment::resolve_workspace_tool_executable(program) { + Ok(tool) => { + resolved = Some(tool); + break; + } + Err(error) => errors.push(error.to_string()), + } + } + let Some(tool) = resolved else { + if binding.required { + return Err(Error::InvalidInput(format!( + "active component `{component_id}` requires one of [{}]: {}", + binding.programs.join(", "), + errors.join("; ") + ))); + } + continue; + }; + if binding.prepend_path { + let parent = tool.path.parent().ok_or_else(|| { + Error::Corrupt(format!( + "resolved tool for component `{component_id}` has no parent directory" + )) + })?; + tool_path_prefixes.push(parent.to_path_buf()); + } + insert_workspace_command_binding( + &mut bindings, + binding.environment, + tool.path.to_string_lossy().into_owned(), + &component_id, + )?; + } + for binding in adapter.cache_command_bindings() { + let Some(root) = self + .active_workspace_cache_path(view, &component_id, binding.cache_name) + .map_err(|error| { + Error::InvalidInput(format!( + "active component `{component_id}` cache `{}` root could not be resolved: {error}", + binding.cache_name + )) + })? + else { + if binding.required { + return Err(Error::Corrupt(format!( + "active component `{component_id}` is missing required cache `{}`", + binding.cache_name + ))); + } + continue; + }; + let path = if binding.relative_path.is_empty() { + root + } else { + safe_join(&root, binding.relative_path).map_err(|error| { + Error::InvalidInput(format!( + "active component `{component_id}` cache `{}` binding `{}` could not be resolved beneath `{}`: {error}", + binding.cache_name, + binding.relative_path, + root.display() + )) + })? + }; + fs::create_dir_all(&path)?; + insert_workspace_command_binding( + &mut bindings, + binding.environment, + path.to_string_lossy().into_owned(), + &component_id, + )?; + } + for binding in adapter.output_command_bindings() { + let output = + self.active_workspace_output_binding(view, &component_id, binding.output_name)?; + let Some((mount_path, policy)) = output else { + if binding.required { + return Err(Error::Corrupt(format!( + "active component `{component_id}` is missing required output `{}`", + binding.output_name + ))); + } + continue; + }; + let root = if binding.direct { + match policy.as_str() { + "immutable_seed_private" => { + self.prepare_direct_private_seed(view, &mount_path)? + } + "writable_private" | "disposable" => { + let path = safe_join(Path::new(&view.generated_upper), &mount_path)?; + fs::create_dir_all(&path)?; + path + } + other => { + return Err(Error::Corrupt(format!( + "component `{component_id}` output `{}` cannot bind policy `{other}` directly", + binding.output_name + ))); + } + } + } else { + let relative = if binding.relative_path.is_empty() { + mount_path.clone() + } else { + format!("{mount_path}/{}", binding.relative_path) + }; + safe_join(Path::new(&view.mountpoint), &relative).map_err(|error| { + Error::InvalidInput(format!( + "active component `{component_id}` output `{}` binding `{relative}` could not be resolved beneath mounted workspace `{}`: {error}", + binding.output_name, view.mountpoint + )) + })? + }; + let path = if !binding.direct || binding.relative_path.is_empty() { + root + } else { + safe_join(&root, binding.relative_path)? + }; + if binding.prepend_path { + output_path_prefixes.push(path.clone()); + } + if let Some(environment) = binding.environment { + insert_workspace_command_binding( + &mut bindings, + environment, + path.to_string_lossy().into_owned(), + &component_id, + )?; + } + } + } + if !output_path_prefixes.is_empty() || !tool_path_prefixes.is_empty() { + // Project-local output executables (for example node_modules/.bin + // or a virtual environment's bin directory) must win over global + // tool directories while the resolved tools still win over the + // ambient PATH. + let mut paths = output_path_prefixes; + paths.extend(tool_path_prefixes); + if let Some(existing) = std::env::var_os("PATH") { + paths.extend(std::env::split_paths(&existing)); + } + let joined = std::env::join_paths(paths).map_err(|error| { + Error::InvalidInput(format!( + "cannot construct managed command PATH from active environment outputs: {error}" + )) + })?; + bindings.insert("PATH".to_string(), joined.to_string_lossy().into_owned()); + } + Ok(bindings.into_iter().collect()) + } + + fn active_workspace_output_binding( + &self, + view: &LaneWorkspaceViewReport, + component_id: &str, + output_name: &str, + ) -> Result> { + self.conn + .query_row( + "SELECT o.mount_path, o.policy + FROM environment_view_generations active + JOIN environment_generation_outputs o + ON o.generation_id = active.generation_id + WHERE active.view_id = ?1 AND o.component_id = ?2 + AND o.output_name = ?3", + params![&view.view_id, component_id, output_name], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional() + .map_err(Into::into) + } + fn active_workspace_cache_path( &self, view: &LaneWorkspaceViewReport, component_id: &str, cache_name: &str, ) -> Result> { - let namespace = self + let cache = self .conn .query_row( - "SELECT c.namespace_id + "SELECT c.namespace_id, n.storage_path FROM environment_view_generations active JOIN environment_generation_caches c ON c.generation_id = active.generation_id + JOIN environment_cache_namespaces n + ON n.namespace_id = c.namespace_id WHERE active.view_id = ?1 AND c.component_id = ?2 AND c.cache_name = ?3", params![&view.view_id, component_id, cache_name], - |row| row.get::<_, String>(0), + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), ) .optional()?; - namespace - .map(|namespace| safe_join(&self.db_dir.join("cache/namespaces"), &namespace)) - .transpose() + let Some((namespace, storage_path)) = cache else { + return Ok(None); + }; + let namespace_root = self.db_dir.join("cache/namespaces"); + fs::create_dir_all(&namespace_root)?; + let expected = safe_join(&namespace_root, &namespace)?; + if let Ok(metadata) = fs::symlink_metadata(&expected) + && (!metadata.is_dir() || metadata.file_type().is_symlink()) + { + return Err(Error::InvalidInput(format!( + "environment cache namespace `{namespace}` is not a real directory" + ))); + } + fs::create_dir_all(&expected)?; + let canonical_storage = Path::new(&storage_path).canonicalize()?; + if canonical_storage != expected { + return Err(Error::Corrupt(format!( + "environment cache namespace `{namespace}` has invalid storage path `{storage_path}`" + ))); + } + let updated = self.conn.execute( + "UPDATE environment_cache_namespaces SET last_used_at = ?1 WHERE namespace_id = ?2", + params![now_ts(), namespace], + )?; + if updated != 1 { + return Err(Error::Corrupt(format!( + "environment cache namespace `{namespace}` disappeared while preparing command bindings" + ))); + } + Ok(Some(expected)) } fn prepare_direct_private_seed( @@ -1681,6 +1962,34 @@ fn checkpoint_view(source_upper: &Path) -> Result { Ok(journal.cut()) } +fn insert_workspace_command_binding( + bindings: &mut BTreeMap, + name: &str, + value: String, + component_id: &str, +) -> Result<()> { + let valid_name = !name.is_empty() + && !name.as_bytes()[0].is_ascii_digit() + && name + .bytes() + .all(|byte| byte == b'_' || byte.is_ascii_alphanumeric()); + if !valid_name || value.contains('\0') { + return Err(Error::Corrupt(format!( + "active component `{component_id}` declares invalid command environment binding `{name}`" + ))); + } + if let Some(existing) = bindings.get(name) { + if existing != &value { + return Err(Error::InvalidInput(format!( + "active component `{component_id}` command environment binding `{name}` conflicts with another component" + ))); + } + return Ok(()); + } + bindings.insert(name.to_string(), value); + Ok(()) +} + fn runtime_service_environment_segment(name: &str) -> String { let mut segment = name .chars() @@ -2694,6 +3003,18 @@ mod tests { "unmounted" ); + let mut leaked = db.acquire_workspace_mount_lease("lease", &backend).unwrap(); + leaked.mark_mounted().unwrap(); + std::mem::forget(leaked); + db.release_current_process_workspace_mount_lease( + &db.lane_workspace_view("lease").unwrap().unwrap().view_id, + ) + .unwrap(); + let released = db.lane_workspace_view("lease").unwrap().unwrap(); + assert_eq!(released.status, "unmounted"); + assert_eq!(released.owner_pid, None); + assert_eq!(released.owner_start_token, None); + let view = db.lane_workspace_view("lease").unwrap().unwrap(); db.conn .execute( @@ -2860,7 +3181,7 @@ mod tests { } #[test] - fn rust_environment_shares_downloads_and_compiler_cache_but_not_target_state() { + fn workspace_environment_does_not_inject_inactive_framework_bindings() { let temp = tempfile::tempdir().unwrap(); fs::write( temp.path().join("Cargo.toml"), @@ -2878,61 +3199,38 @@ mod tests { } else { LaneWorkdirMode::FuseCow }; - for lane in ["cargo-a", "cargo-b"] { - db.spawn_lane_with_workdir_mode_paths_and_neighbors( - lane, - Some("main"), - mode.clone(), - None, - None, - None, - &[], - false, - ) - .unwrap(); + db.spawn_lane_with_workdir_mode_paths_and_neighbors( + "inactive", + Some("main"), + mode, + None, + None, + None, + &[], + false, + ) + .unwrap(); + let view = db.lane_workspace_view("inactive").unwrap().unwrap(); + let root = db + .get_ref(&db.lane_branch("inactive").unwrap().ref_name) + .unwrap() + .root_id; + let environment = db + .workspace_command_environment(&view, &root) + .unwrap() + .into_iter() + .collect::>(); + for name in [ + "CARGO_HOME", + "CARGO_TARGET_DIR", + "GOCACHE", + "GOMODCACHE", + "npm_config_cache", + "VIRTUAL_ENV", + "TRAIL_CMAKE_BUILD_DIR", + ] { + assert!(!environment.contains_key(name), "unexpected binding {name}"); } - let environment = |lane: &str| { - let view = db.lane_workspace_view(lane).unwrap().unwrap(); - let root = db - .get_ref(&db.lane_branch(lane).unwrap().ref_name) - .unwrap() - .root_id; - db.workspace_command_environment(&view, &root) - .unwrap() - .into_iter() - .collect::>() - }; - let a = environment("cargo-a"); - let b = environment("cargo-b"); - assert_eq!(a["CARGO_HOME"], b["CARGO_HOME"]); - assert_eq!(a["SCCACHE_DIR"], b["SCCACHE_DIR"]); - assert_ne!(a["CARGO_TARGET_DIR"], b["CARGO_TARGET_DIR"]); - assert_eq!( - Path::new(&a["CARGO_TARGET_DIR"]).parent(), - Some(Path::new( - &db.lane_workspace_view("cargo-a") - .unwrap() - .unwrap() - .generated_upper - )) - ); - assert_eq!( - Path::new(&b["CARGO_TARGET_DIR"]).parent(), - Some(Path::new( - &db.lane_workspace_view("cargo-b") - .unwrap() - .unwrap() - .generated_upper - )) - ); - assert_eq!( - Path::new(&a["CARGO_TARGET_DIR"]).file_name(), - Some(std::ffi::OsStr::new("target")) - ); - assert_eq!( - Path::new(&b["CARGO_TARGET_DIR"]).file_name(), - Some(std::ffi::OsStr::new("target")) - ); } #[test] diff --git a/trail/tests/e2e.rs b/trail/tests/e2e.rs index d8469a13..f457fc4d 100644 --- a/trail/tests/e2e.rs +++ b/trail/tests/e2e.rs @@ -12473,7 +12473,7 @@ fn environment_sync_reuses_one_node_layer_across_http_and_mcp_parity() { assert_eq!(status[0]["status"], "ready"); assert_eq!( status[0]["adapter"]["distribution_digest"], - "builtin:node-plan-v1" + "builtin:node-plan-v2" ); let generation = trail::server::handle_http_request( @@ -13312,7 +13312,7 @@ fn environment_sync_reuses_one_node_layer_across_http_and_mcp() { assert_eq!(status[0]["status"], "ready"); assert_eq!( status[0]["adapter"]["distribution_digest"], - "builtin:node-plan-v1" + "builtin:node-plan-v2" ); } diff --git a/trail/tests/lane_environment_inheritance.rs b/trail/tests/lane_environment_inheritance.rs index e02bf790..3764dc8b 100644 --- a/trail/tests/lane_environment_inheritance.rs +++ b/trail/tests/lane_environment_inheritance.rs @@ -70,7 +70,7 @@ fn lane_fork_inherits_verified_immutable_layer_with_fresh_private_uppers() { "INSERT INTO environment_component_states( view_id,component_id,adapter_identity,adapter_version,implementation_version, distribution_digest,kind,expected_key,attached_key,status,reason,updated_at) - VALUES(?1,'node','trail/node@1',1,?2,'builtin:node-plan-v1','dependency', + VALUES(?1,'node','trail/node@1',1,?2,'builtin:node-plan-v2','dependency', ?3,?3,'ready',NULL,1)", params![ &parent_view.view_id,