diff --git a/CHANGELOG.md b/CHANGELOG.md index 08a2135f5d..1177e7d5be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,36 @@ All notable changes to this project will be documented in this file. Effect 4 instance through a peer dependency instead of bundling a private Effect runtime, preventing cross-major schema AST crashes in LiveStore Devtools. +- **pnpm / dependency identity**: make pnpm the explicit authority for + live package dependency edges. Dependency projection and repair may no + longer invent nested links by scanning the store or choosing a target by + package name; declared compatibility extensions continue through generated + `packageExtensions`. Live virtual topology now lives exclusively under each + Materialization Root's `node_modules/.pnpm`. Repair discards one root-local + graph and reinvokes its + canonical pnpm install, eliminating shared graph registries and coordinated + multi-root repair. The complete disposable pnpm Store Cache may be shared + inside one same-user trust boundary, but it never owns dependency edges. + Prepared workspace normalization now relinks injected + packages only through pnpm's exact locator mapping. Record the mixed Effect 3 + / Effect 4 counterexample and + add coverage for root-local graph authority and the supported extension path. + Replace the flat dependency-materialization glossary with a federated + ontology that separates root-owned state, profile identity, graph authority, + projections, and storage policy. Remove the unused live profile artifact and + storage presets; Nix prepared-dependency and Buck2 evidence retain the existing + `profileKey` compatibility boundary. Remove the GVS-only + `enableGlobalVirtualStore` and `gvsTypeExtensions` generator APIs. +- **pnpm / cross-worktree cache reuse**: share pnpm's complete disposable Store + Cache (content-addressed files plus its derived index) across mutually trusted + local worktrees while keeping GVS disabled and every dependency graph under + its own root. Let pnpm's native `auto` import policy select clone, hardlink, or + copy, fail closed on Linux when the cache cannot provide same-device zero-copy + reuse, keep CI caches job-local, and leave Nix prepared dependencies on their + independent content-addressed path. Root repair never prunes the host cache; + focused two-root tests prove zero-download reuse, distinct virtual stores, + offline rematerialization, native-package isolation, and the explicit + same-user hardlink trust boundary. - **devenv cli-guard ownership**: drop the remaining self-consumer `lib.lowPrio effectTsgo` / `lib.lowPrio pnpmPkg` boilerplate. Passing `tsBinPkg` / `pnpmPkg` to the task modules is sufficient because the guards diff --git a/context/dependency-materialization/.decisions/0001-effect-utils-owns-dependency-materialization-vrs.md b/context/dependency-materialization/.decisions/0001-effect-utils-owns-dependency-materialization-vrs.md index 5b781f87c9..8073751c11 100644 --- a/context/dependency-materialization/.decisions/0001-effect-utils-owns-dependency-materialization-vrs.md +++ b/context/dependency-materialization/.decisions/0001-effect-utils-owns-dependency-materialization-vrs.md @@ -1,17 +1,15 @@ # 0001: effect-utils owns dependency materialization VRS -## Decision +Status: accepted -effect-utils owns the reusable dependency materialization VRS hierarchy. The -canonical docs live under `context/dependency-materialization/` and cover live -pnpm materialization, projection, Nix prepared dependencies, store authority, -Buck2 evidence, and producer observability. +## Context -dotfiles keeps fleet orchestration, local runner policy, and repo-alignment -guidance. It does not keep parallel VRS roots for reusable pnpm/Nix dependency -contracts. +Reusable pnpm/Nix dependency tooling lives in effect-utils while its largest +fleet consumer and earlier research lived in dotfiles. Without one intent owner, +the same materialization behavior accumulated competing terminology, profiles, +and repair policies in both repositories. -## Rationale +## Evidence and Argument The implementation and reusable public API live in effect-utils. Keeping the VRS in dotfiles would make private orchestration policy the source of truth for @@ -22,6 +20,25 @@ The hierarchy also matches the system shape better than two flat documents: one root contract defines identity and authority vocabulary, while child VRS nodes refine each realization. +## Options + +| Option | Tradeoffs | +| ------------------------------------- | -------------------------------------------------------------------------------------------------------- | +| effect-utils owns reusable DMP intent | Co-locates contract and implementation; downstream fleet docs must reference rather than restate it. | +| dotfiles owns DMP intent | Keeps the original research location but makes private orchestration authoritative for reusable tooling. | +| duplicate synchronized VRS roots | Local convenience at the cost of inevitable drift and ambiguous authority. | + +## Decision + +effect-utils owns the reusable dependency materialization VRS hierarchy. The +canonical docs live under `context/dependency-materialization/` and cover live +pnpm materialization, projection, Nix prepared dependencies, store authority, +Buck2 evidence, and producer observability. + +dotfiles keeps fleet orchestration, local runner policy, and repo-alignment +guidance. It does not keep parallel VRS roots for reusable pnpm/Nix dependency +contracts. + ## Consequences - effect-utils specs must stay current with implementation changes to pnpm diff --git a/context/dependency-materialization/.decisions/0002-effect-utils-owned-bin-projection.md b/context/dependency-materialization/.decisions/0002-effect-utils-owned-bin-projection.md index b5e9f1ef70..1f58015bfd 100644 --- a/context/dependency-materialization/.decisions/0002-effect-utils-owned-bin-projection.md +++ b/context/dependency-materialization/.decisions/0002-effect-utils-owned-bin-projection.md @@ -1,6 +1,6 @@ # 0002: effect-utils-owned bin projection -Status: Accepted +Status: accepted ## Context @@ -10,6 +10,23 @@ exclude `.bin` entirely. The projection layer needs pnpm-compatible executable links without making pnpm lifecycle execution or install-time side effects part of the trust boundary. +## Evidence and Argument + +- Strict lifecycle-disabled installs can leave missing executable projections. +- Prepared dependency artifacts deliberately exclude `.bin`, so projection must + be recreated rather than archived as dependency data. +- pnpm's published linker remains useful as a compatibility oracle, but making + it runtime authority would couple the stable DMP surface to pnpm internals and + Node engine constraints. + +## Options + +| Option | Tradeoffs | +| -------------------------------- | ------------------------------------------------------------------------------- | +| effect-utils pure projector | Stable lifecycle-free authority with explicit compatibility responsibility. | +| pnpm linker as runtime authority | Maximum upstream behavior reuse but imports unstable internal/runtime coupling. | +| lifecycle-generated bins | Delegates behavior but violates the purity boundary. | + ## Decision Effect-utils owns the production bin projector. @@ -22,17 +39,6 @@ emits projection reports. pnpm's published bin-linking packages are used as conformance oracles in tests, not as the runtime authority. -## Rationale - -- pnpm's current linker package is small but pulls in pnpm internals, logging, - manifest readers, workspace readers, command-shim code, and Node engine - constraints. -- The effect-utils boundary needs a stable projection contract independent of - pnpm's install implementation details. -- pnpm behavior still matters for compatibility. The conformance fixture keeps - scoped command names, `directories.bin`, path-safety checks, conflict - behavior, and missing-target handling visible. - ## Consequences - The implementation must cover pnpm-compatible bin edge cases intentionally diff --git a/context/dependency-materialization/.decisions/0003-native-policy-pure-package-artifact.md b/context/dependency-materialization/.decisions/0003-native-policy-pure-package-artifact.md index 5b62c0f070..d34c9d0a54 100644 --- a/context/dependency-materialization/.decisions/0003-native-policy-pure-package-artifact.md +++ b/context/dependency-materialization/.decisions/0003-native-policy-pure-package-artifact.md @@ -1,6 +1,6 @@ # 0003 Native Policy Uses Pure Package Artifact -Status: **Accepted** +Status: accepted ## Context @@ -9,6 +9,22 @@ may remain dependency data without lifecycle execution. The implementation used `fod-accepted-prebuilt`, which tied the public classification to one current realization: Nix fixed-output prepared dependencies. +## Evidence and Argument + +- The public DMP contract spans live pnpm, Nix, CI, and future Buck2 evidence; + `fod-accepted-prebuilt` incorrectly named one current realization. +- Current accepted prebuilts are still locked and scanned by prepared-deps + policy, so the broader name does not weaken the purity gate. +- The term aligns with DMP-R04 and DMP.NIX.NATIVE-R03. + +## Options + +| Option | Tradeoffs | +| ---------------------------- | ---------------------------------------------------------------------------------- | +| `pure-package-artifact` | Names the cross-realization property; requires specs to state each concrete proof. | +| `fod-accepted-prebuilt` | Mechanically precise today but leaks Nix FOD realization into the public ontology. | +| one generic native exception | Simpler vocabulary but erases the purity/build distinction. | + ## Decision Use `pure-package-artifact` as the canonical native dependency policy tag. @@ -21,15 +37,6 @@ Native package families are classified as: | `pure-package-artifact` | Package contents are accepted as data without lifecycle execution. | | `denied-lifecycle-build` | Package requires scripts/builds and is rejected until integrated. | -## Rationale - -- The DMP contract spans live pnpm, Nix prepared deps, CI jobs, and Buck2 - evidence. A public tag should describe the dependency-materialization - boundary, not only the fixed-output derivation mechanism. -- Current accepted prebuilts are still locked and scanned by prepared-deps - policy; that mechanism belongs in the owning spec and implementation details. -- The term matches DMP-R04 and DMP.NIX.NATIVE-R03. - ## Consequences - Audit output now asks new gated native package families to be classified as diff --git a/context/dependency-materialization/.decisions/0004-strict-prepared-scan-v18.md b/context/dependency-materialization/.decisions/0004-strict-prepared-scan-v18.md index 94ae703337..db26674a53 100644 --- a/context/dependency-materialization/.decisions/0004-strict-prepared-scan-v18.md +++ b/context/dependency-materialization/.decisions/0004-strict-prepared-scan-v18.md @@ -1,6 +1,6 @@ # 0004 Strict Prepared Scan Uses One Version Bump -Status: **Accepted** +Status: accepted ## Context @@ -10,6 +10,23 @@ outputs, and unclassified platform package directories should fail the prepared artifact scan. Removing `.bin` changes recursive output hashes, so the transition necessarily creates fixed-output hash churn. +## Evidence and Argument + +- Prepared artifacts are dependency data; archived `.bin`, package-manager + state, and unclassified native output violate that boundary. +- Removing `.bin` necessarily changes recursive fixed-output hashes, making the + transition versioned regardless of rollout shape. +- Projection and native output already have separate owners, so a parallel + lenient policy would preserve ambiguity rather than compatibility. + +## Options + +| Option | Tradeoffs | +| ------------------------------- | --------------------------------------------------------------------------- | +| one strict v18 boundary | Converges immediately with mechanical hash churn. | +| report-only transition | Reduces initial disruption but permits known-impure artifacts indefinitely. | +| parallel strict/legacy profiles | Supports gradual adoption but doubles policy and hash authority. | + ## Decision Use one convergent prepared artifact version bump for the strict scan @@ -26,15 +43,6 @@ The next strict prepared-deps purity transition: Do not introduce a report-only phase, and do not keep old and new scan policies active behind profile gates once `v18` lands. -## Rationale - -- Prepared dependency artifacts are data artifacts. Carrying a lenient legacy - scan beside the strict scan would keep the most important ambiguity alive. -- The hash churn is real but mechanical. It is better handled as an explicit - versioned boundary than as piecemeal report-only drift. -- Projection and native output ownership are already modeled separately, so the - strict scan is the clearest convergence point for the Nix-prepared realization. - ## Consequences - The implementation milestone that lands strict scan enforcement must also diff --git a/context/dependency-materialization/.decisions/0005-fod-repair-targets-are-eval-metadata.md b/context/dependency-materialization/.decisions/0005-fod-repair-targets-are-eval-metadata.md index 024c754eb3..96ea04d0c7 100644 --- a/context/dependency-materialization/.decisions/0005-fod-repair-targets-are-eval-metadata.md +++ b/context/dependency-materialization/.decisions/0005-fod-repair-targets-are-eval-metadata.md @@ -1,6 +1,6 @@ # 0005 FOD Repair Targets Are Eval Metadata -Status: **Accepted** +Status: accepted ## Context @@ -9,6 +9,22 @@ evidence. A committed per-target witness file would make evidence visible in source, but it would duplicate package Nix metadata, the declared hash, the install root, and the profile identity already available through evaluation. +## Evidence and Argument + +- Nix evaluation already exposes the asserted hash, derivation, install root, + profile identity, and freshness inputs at the owning boundary. +- A checked-in witness would duplicate those fields and introduce another stale + authority. +- Cross-system measurement is run evidence unavailable at pure evaluation time. + +## Options + +| Option | Tradeoffs | +| ------------------------------------------- | ----------------------------------------------------------------------- | +| evaluated repair metadata plus run evidence | One committed authority with operational measurement kept truthful. | +| per-target witness files | Easy source review but duplicates Nix metadata and drifts. | +| source parsing only | Avoids a producer contract but is brittle and loses evaluated identity. | + ## Decision Expose FOD hash repair targets as evaluated package metadata, and keep measured @@ -23,17 +39,6 @@ record measured outputs as generated run evidence. Do not add checked-in JSON, YAML, or Markdown witness files per prepared dependency target. -## Rationale - -- Nix fixed-output derivations already place the asserted hash at the - derivation boundary; a parallel source file is another stale authority. -- Evaluated metadata can include the profile key, install root, declared hash, - derivation path, freshness inputs, and update path without asking package - authors to maintain another artifact. -- Cross-system measurement is an operation, not a static fact available at - evaluation time. Keeping it in run evidence prevents accidental shared-hash - collapse while avoiding source churn. - ## Consequences - Repair tools should discover prepared-deps targets through evaluated package diff --git a/context/dependency-materialization/.decisions/0006-pure-reuse-with-root-local-graph-authority.md b/context/dependency-materialization/.decisions/0006-pure-reuse-with-root-local-graph-authority.md new file mode 100644 index 0000000000..c4ce2caca8 --- /dev/null +++ b/context/dependency-materialization/.decisions/0006-pure-reuse-with-root-local-graph-authority.md @@ -0,0 +1,85 @@ +# 0006 Pure Reuse With Root-Local Graph Authority + +Status: accepted + +## Context + +Local development must reuse dependency bytes across many worktrees without +turning shared storage into dependency identity, lifecycle, or repair authority. +pnpm can share either its Store Cache alone or also its Global Virtual Store. +The latter may reuse more topology state but expands the writable/failure scope +across otherwise independent roots. + +## Evidence and Argument + +- The [mixed Effect-generation experiment](../01-live-pnpm/.experiments/2026-07-17-shared-gvs-identity-and-repair.md) + proved that native shared GVS preserved correct Effect and peer-context + identities in both install orders. It also proved that `pnpm install --force` + did not repair a missing shared GVS edge; repair required discarding shared + `links/` state. +- The committed [default-gate evidence](../07-verification/evidence/storage-sharing-default-v2.json) + proves material package-byte and file-count reuse across real Linux/ext4 and + Darwin/APFS workloads. +- The two-root shared-cache fixture proves zero second-root downloads, offline + rematerialization, concurrent cold/offline roots, distinct native-package + inodes, and distinct virtual stores. +- Nix prepared dependencies already demonstrate the stronger reusable-unit + shape: declared inputs produce immutable, integrity-addressed output without + lifecycle mutation or ambient live-store authority. + +The missing evidence is a same-workload comparison of root-local topology with +shared and identity-partitioned GVS. Current pnpm GVS options also fail the +strict reuse boundary because consumers share mutable topology and repair +state. Therefore this decision records the current pnpm compatibility baseline; it +does not present root-local rematerialization as the long-term ideal. + +## Options + +| Option | Tradeoffs | +| --------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| A. Shared Store Cache with root-local virtual topology | Maximizes proven package-data reuse while keeping graph mutation and repair independently bounded; repeats some topology materialization. | +| B. Shared Store Cache with one shared GVS | May reuse more topology work, but shares writable graph realization and expands one-root repair/fault scope. | +| C. Shared Store Cache with GVS partitioned by declared graph identity | Narrows coupling relative to B but still shares mutable topology within a partition and adds lifecycle complexity. | +| D. Fully isolated stores and topology | Simplest isolation, but discards large proven byte/file-count and second-root reuse gains. | +| E. Hermetic Dependency Artifact | Reuses content and topology by complete declared-input identity with immutable, atomic results; requires a producer, compatibility projection, ownership, and GC contract not exposed by current live pnpm. | + +## Decision + +Keep A as the current pnpm compatibility baseline under DELTA-001, and choose E as the architectural +target. + +Reusable package data must first be deterministic, integrity-addressed, derived +from declared inputs, lifecycle-free, and immutable by contract. Within that +eligible layer, share as broadly as the trust and platform evidence allow. Keep +Dependency Graph, virtual topology, Projection State, and repair authority at +one Materialization Root so reuse never grants mutation authority. + +Move repeated graph/topology work across roots only by replacing mutable shared +state with a Hermetic Dependency Artifact keyed by the complete lock graph, +platform, package-manager policy, and all identity-affecting inputs. Publish it +atomically, mount or project it read-only, and make eviction independent of +consumers. This follows the property that gives Nix stores and hermetic build +action caches broad safe reuse; it does not require inventing a second mutable +package-manager database. + +Use DMP.VER-R12 to quantify A, B, and C and to identify topology work worth +capturing in E. B or C cannot replace A merely by winning a benchmark: a +challenger must first eliminate cross-root mutable topology and repair authority +and pass identity, purity, data-safety, concurrency, and bounded-repair gates. + +## Consequences + +- Managed live pnpm uses a shared whole Store Cache and root-local + `node_modules/.pnpm`; GVS is disabled by the current spec. Sharing the cache's + mutable pnpm index is a transitional compatibility divergence tracked by + [DELTA-001](../.delta/DELTA-001-whole-store-mutable-index.md), not part of the + accepted pure reuse target. +- Direct mutation of imported dependency files and dependency lifecycle scripts + remain outside the managed contract. Native/build-sensitive output is + isolated or supplied as immutable Nix output. +- Root repair discards only root-owned graph/projection state and never invents + edges or sweeps the host Store Cache. +- Current GVS remains a measurement subject, not an admissible end state or a + synonym for cache reuse or runtime identity. +- The long-term design should remove repeated pure topology work by publishing + immutable graph-addressed artifacts, rather than widening mutation scope. diff --git a/context/dependency-materialization/.delta/DELTA-001-whole-store-mutable-index.md b/context/dependency-materialization/.delta/DELTA-001-whole-store-mutable-index.md new file mode 100644 index 0000000000..afc13ad48f --- /dev/null +++ b/context/dependency-materialization/.delta/DELTA-001-whole-store-mutable-index.md @@ -0,0 +1,57 @@ +# DELTA-001: Whole Store Cache shares mutable pnpm indexes + +Status: open + +## Divergence + +The current live-development realization shares one complete pnpm Store Cache, +including pnpm-mutated derived indexes, across Materialization Roots. This gives +measured second-root acceleration but does not satisfy the normative pure +cross-root reusable-state boundary. + +## VRS + +- [DMP-R21 and DMP-R24](../requirements.md) admit only declared-input-derived, + immutable state and hermetic topology work to cross-root reuse. +- [Decision 0006](../.decisions/0006-pure-reuse-with-root-local-graph-authority.md) + selects a Hermetic Dependency Artifact as the architectural target. +- [Store authority](../04-store-authority/requirements.md) keeps mutable + package-manager indexes outside the claimed reusable layer. + +## Implementation + +`nix/devenv-modules/tasks/shared/pnpm.nix` supplies one host Store Cache to +managed local installs. pnpm mutates its derived indexes under cache admission +and maintenance coordination. The committed storage evidence demonstrates that +this whole-store realization avoids second-root downloads. The focused +[split-files prune experiment](../04-store-authority/.experiments/2026-06-22-split-files-pool-prune.md) +proves that mutually invisible indexes over one files pool cannot independently +own destructive prune authority. + +## Resolution Approach + +Replace cross-root mutable index sharing with either: + +1. immutable/integrity-addressed package data plus independently recoverable + writable metadata; or +2. a Hermetic Dependency Artifact that captures reusable graph/topology work by + complete declared-input identity and is consumed read-only. + +A read-only seed with root-local writable overlays is admissible if it preserves +the same purity, atomic publication, independent repair, and maximal-data-reuse +properties. Do not regress to mutually invisible indexes over a destructively +pruned shared files pool. + +## Direction + +update implementation + +## Resolution Signal + +- Cross-root state is immutable after atomic publication and keyed by complete + declared or content identity. +- Each mutable package-manager index can be discarded or repaired without + coordinating an independent Materialization Root. +- Same-workload second-root online/offline, concurrent-root, physical-byte, + inode/file-count, and repair benchmarks meet the DMP.VER-R12 gates. +- The whole-store compatibility path and this delta are removed. diff --git a/context/dependency-materialization/.delta/DELTA-002-prepared-v18-bin-scan.md b/context/dependency-materialization/.delta/DELTA-002-prepared-v18-bin-scan.md new file mode 100644 index 0000000000..f6160ea332 --- /dev/null +++ b/context/dependency-materialization/.delta/DELTA-002-prepared-v18-bin-scan.md @@ -0,0 +1,49 @@ +# DELTA-002: Prepared v18 artifacts still contain bin projections + +Status: open + +## Divergence + +The accepted prepared-v18 contract requires `.bin` projection directories to be +excluded and rejected by the strict scan. Current realized prepared artifacts +still archive dozens of `.bin` directories and the implementation rewrites them +after restore. + +## VRS + +- [Decision 0002](../.decisions/0002-effect-utils-owned-bin-projection.md) + assigns bin projection to the effect-utils projector rather than prepared + dependency data. +- [Decision 0004](../.decisions/0004-strict-prepared-scan-v18.md) requires the + v18 prepared artifact to strip and reject `.bin` immediately. +- [The root spec](../spec.md) classifies `.bin` as Projection State and requires + the strict prepared scan to reject it. + +## Implementation + +`nix/workspace-tools/lib/mk-pnpm-deps.nix` currently strips pnpm bookkeeping but +does not strip or reject `.bin`; its restore path still chmods/rewrites archived +bin projections. Realized `genie-pnpm-deps` and `megarepo-pnpm-deps` artifacts +contained 48 and 51 `.bin` directories respectively, while neither contained +`.modules.yaml`, `.pnpm/lock.yaml`, or native `*.node` files. + +## Resolution Approach + +Make normalization remove `.bin`, make the strict scan fail on any remaining +bin directory or shim, and recreate bins exclusively through the accepted pure +projector after immutable data is realized. Refresh the affected fixed-output +hashes through Evergreen and prove the exact prepared artifacts. + +## Direction + +update implementation + +## Resolution Signal + +- Realized prepared v18 artifacts contain zero `.bin` directories/shims and + pass the strict scan. +- Restore no longer chmods or rewrites archived bins. +- The pure projector recreates all expected scoped, aliased, package-local, and + platform-correct bins from immutable package manifests. +- Exact prepared builds and downstream CLI consumers pass on Linux and Darwin. +- This delta is removed. diff --git a/context/dependency-materialization/01-live-pnpm/.experiments/2026-06-24-ci-cache-boundary.md b/context/dependency-materialization/01-live-pnpm/.experiments/2026-06-24-ci-cache-boundary.md new file mode 100644 index 0000000000..c21490d698 --- /dev/null +++ b/context/dependency-materialization/01-live-pnpm/.experiments/2026-06-24-ci-cache-boundary.md @@ -0,0 +1,27 @@ +# 2026-06-24 CI Cache Boundary + +## Question + +Is the pnpm Store Cache alone the best warm-install cache boundary for the +measured pnpm 11 and GVS realization? + +## Method + +Compared warm installs after restoring only the Store Cache with installs after +restoring the measured full pnpm hot-state boundary. + +## Result + +Restoring pnpm home reused more of the hot path than restoring only the Store +Cache in the measured GVS realization. + +## Conclusion + +The Store Cache alone was not the complete hot-state boundary in that historical +realization. Current CI remains job-local, and any future reusable artifact must +derive its boundary from declared identity rather than ambient pnpm home state. + +## VRS Impact + +Supports DMP.LIVE-R10's job-local CI boundary and DMP-R24's requirement for a +declared Hermetic Dependency Artifact identity. diff --git a/context/dependency-materialization/01-live-pnpm/.experiments/2026-06-24-runner-local-seed.md b/context/dependency-materialization/01-live-pnpm/.experiments/2026-06-24-runner-local-seed.md new file mode 100644 index 0000000000..7a727d9cb2 --- /dev/null +++ b/context/dependency-materialization/01-live-pnpm/.experiments/2026-06-24-runner-local-seed.md @@ -0,0 +1,28 @@ +# 2026-06-24 Runner-Local Seed + +## Question + +Can a runner-local seed of package content plus job-local writable metadata +preserve isolation and reduce install time? + +## Method + +Compared synthetic hardlink reuse with integrated real-workspace installs and +tested the seed across the explored pnpm store-version boundary. + +## Result + +Synthetic hardlink results were promising, but integrated installs were slower +than the warm baseline and the seed exposed pnpm store-version portability +problems. + +## Conclusion + +The explored seed was rejected. A new immutable-seed design must prove complete +identity, pnpm-version compatibility, real-workload latency, parallel safety, +and actual immutability rather than assuming a hardlink is read-only. + +## VRS Impact + +Constrains DMP-R24 and DELTA-001 resolution without rejecting a differently +constructed, atomically published immutable seed. diff --git a/context/dependency-materialization/01-live-pnpm/.experiments/2026-06-24-setup-fan-out-archive.md b/context/dependency-materialization/01-live-pnpm/.experiments/2026-06-24-setup-fan-out-archive.md new file mode 100644 index 0000000000..cae5cf16e9 --- /dev/null +++ b/context/dependency-materialization/01-live-pnpm/.experiments/2026-06-24-setup-fan-out-archive.md @@ -0,0 +1,25 @@ +# 2026-06-24 Setup Fan-Out Archive + +## Question + +Can one setup job install once, archive prepared live state, and fan it out more +cheaply than each sibling job running a warm install? + +## Method + +Compared integrated archive pack/restore with independent warm installs on the +explored self-hosted runner shape. + +## Result + +Archive pack and restore cost outweighed the warm-install savings. + +## Conclusion + +Live setup/fan-out must beat the current warm path in integrated real-workload +benchmarks, not only synthetic copy tests. + +## VRS Impact + +Supports DMP.STORE-R14 and DMP.VER-R12's same-workload, multidimensional default +gate. diff --git a/context/dependency-materialization/01-live-pnpm/.experiments/2026-07-17-shared-gvs-identity-and-repair.md b/context/dependency-materialization/01-live-pnpm/.experiments/2026-07-17-shared-gvs-identity-and-repair.md new file mode 100644 index 0000000000..982d8b1dc7 --- /dev/null +++ b/context/dependency-materialization/01-live-pnpm/.experiments/2026-07-17-shared-gvs-identity-and-repair.md @@ -0,0 +1,57 @@ +# 2026-07-17 Shared GVS Identity And Repair + +## Question + +Does sharing one pnpm Global Virtual Store across independently locked Effect 3 +and Effect 4 roots make dependency identity install-order-dependent, and what +repair scope is required after a shared GVS edge is damaged? + +## Method + +- Installed an Effect 3 root containing `effect-distributed-lock` and an Effect + 4 root into one pnpm 11.3 GVS in both orders. +- Compared shared GVS with profile-isolated GVS and workspace-local virtual + stores while retaining package-content reuse. +- Repeated the peer-context case with `react-redux@9.2.0` against React 18 and + React 19. +- Reproduced the downstream name-only repair traversal separately. +- Removed a selected GVS edge and compared `pnpm install --force` with discard + and rematerialization. +- Removed only the manual repair traversal in the real dotfiles/Vista workspace + and reran precise typechecks. + +## Result + +- Native shared GVS passed both install orders. pnpm kept + `effect-distributed-lock` linked to Effect 3.21.4 and retained distinct React + peer contexts. +- The out-of-band name-only repair selected Effect 4.0.0-beta.97 and redirected + peer consumers, producing the observed identity/type failures. +- A declared `packageExtensions` edge remained represented in lock state; + synthesizing an undeclared filesystem link falsified dependency truth. +- `pnpm install --force` reused an incomplete GVS instance and did not restore + its missing edge. Discarding the root projection and shared GVS `links/`, while + retaining content-addressed package files, restored it. +- Removing the manual repair traversal kept the real shared-GVS install green, + restored precise Vista/effect-utils typechecks, and completed the linked-repo + install in 22.2 seconds. + +## Conclusion + +Shared GVS was not the cause of the Effect identity failure; the secondary +name-only graph writer was. The experiment did not show a correctness advantage +for local virtual stores in the tested normal-operation case. It did show that +shared GVS expands damaged-topology repair beyond one Materialization Root. + +Root-local virtual topology was subsequently selected as the safety-biased +default for authority and repair containment, not because this experiment +proved it globally faster or smaller. That optimization claim remains pending a +direct topology-reuse comparison. + +## VRS Impact + +- Supports DMP.LIVE-R07 and DMP.STORE-R02 by proving pnpm must remain the sole + Dependency Edge writer and repair must discard owned state. +- Supports decision 0006's bounded-authority rationale. +- Leaves DMP.VER-R12 open: shared, identity-partitioned, and root-local topology + still need same-workload physical-byte and latency comparison. diff --git a/context/dependency-materialization/01-live-pnpm/experiments.md b/context/dependency-materialization/01-live-pnpm/experiments.md deleted file mode 100644 index f98c6c8aae..0000000000 --- a/context/dependency-materialization/01-live-pnpm/experiments.md +++ /dev/null @@ -1,57 +0,0 @@ -# Live pnpm Experiments - -This file records non-normative live pnpm evidence. Normative behavior lives in -[spec.md](./spec.md). - -## CI Cache Boundary - -Hypothesis: - -- CI should cache the pnpm store directory as the primary warm-install - boundary. - -Result: - -- Rejected for the measured pnpm 11 + GVS model. Warm installs reused more of - the hot path when the pnpm home was restored than when only the store - directory was restored. - -Conclusion: - -- CI cache policy should describe the full profile hot state instead of - assuming `store-dir` alone is the reusable boundary. - -## Setup/Fan-Out Archive - -Hypothesis: - -- One setup job can install once, archive the prepared live state, and fan it - out to sibling jobs more cheaply than each sibling running a warm install. - -Result: - -- Rejected for the explored self-hosted runner shape. Archive pack and restore - cost outweighed the warm install savings. - -Conclusion: - -- Live setup/fan-out must beat the current warm path in integrated benchmarks, - not only in synthetic copy tests. - -## Runner-Local Seed - -Hypothesis: - -- A runner-local seed of shared package content plus job-local metadata can - preserve isolation and reduce install time. - -Result: - -- Rejected in the explored implementation. Synthetic hardlink tests looked - promising, but integrated install benchmarks were slower than the current - warm path and exposed pnpm store-version portability issues. - -Conclusion: - -- Future seed traits need real-workspace benchmarks, parallel stress, and - pnpm-version portability proof before becoming defaults. diff --git a/context/dependency-materialization/01-live-pnpm/requirements.md b/context/dependency-materialization/01-live-pnpm/requirements.md index a611ab4b99..f1f48435d5 100644 --- a/context/dependency-materialization/01-live-pnpm/requirements.md +++ b/context/dependency-materialization/01-live-pnpm/requirements.md @@ -7,19 +7,22 @@ Live pnpm materialization is the mutable worktree realization of the development, devenv tasks, and CI jobs that run against a checked-out workspace. -The live model is the authority for workspace topology, install ownership, and -runtime dependency identity. Nix prepared dependency artifacts, Buck2 evidence, -and observability derive from this model instead of inventing a parallel -dependency graph. +The live model names the authoritative workspace topology, Materialization +Root, and Authoritative Materializer. Nix prepared dependency artifacts, Buck2 +evidence, and observability derive from this model instead of inventing a +parallel Dependency Graph. ## Assumptions - **A01 DMP base:** This subsystem refines DMP-R01 through DMP-R19. - **A02 Canonical topology:** A managed install receives one selected workspace - topology and one install owner before pnpm is invoked. -- **A03 Mutable realization:** Live `node_modules`, pnpm metadata, GVS links, - and package projections are mutable profile state, not immutable dependency - artifacts. + topology and one Materialization Root before pnpm is invoked as its + Authoritative Materializer. +- **A03 Mutable realization:** Live `node_modules`, its root-local virtual + store, graph metadata, and package projections are mutable Materialization + Root state, not immutable dependency artifacts or state owned by a + Materialization Profile. A pnpm-owned Store Cache is separate, disposable + state and is not graph authority. ## Acceptable Tradeoffs @@ -27,17 +30,17 @@ dependency graph. rather than accepting arbitrary package-directory installs. - **T02 Job-local CI:** CI may give up host-wide mutable-state reuse to keep concurrent jobs isolated. -- **T03 Shared content with local metadata:** Local development may share - package content across profiles only when metadata, projection, repair, and - GC authority remain explicit. +- **T03 Shared cache with local graphs:** Mutually trusted local development + roots may share a pnpm Store Cache while keeping graph, projection, and + repair authority root-local. ## Requirements ### Must preserve topology authority -- **DMP.LIVE-R01 Install owner:** Every live install must name its owner root - and topology. Package-local or sibling-root installs must not become - authoritative implicitly. +- **DMP.LIVE-R01 Materialization Root:** Every live install must name its + Materialization Root, Authoritative Materializer, and topology. Package-local + or sibling-root installs must not become authoritative implicitly. Refines: DMP-R11. - **DMP.LIVE-R02 Standalone validity:** A repo must remain installable through its own canonical workspace topology even when it is also consumed from a @@ -48,21 +51,26 @@ dependency graph. lockfiles as a side effect. Refines: DMP-R11. - **DMP.LIVE-R04 Nested roots:** Nested authoritative install roots must be - modeled as explicit managed install roots with separate profile state. - Refines: DMP-R09, DMP-R11, DMP-R12. + modeled as explicit Materialization Roots with separate mutable realization + state and root-local virtual topology. Equivalent roots may share immutable + content bytes, but not writable dependency graph state. + Refines: DMP-R11, DMP-R12. ### Must preserve dependency identity -- **DMP.LIVE-R05 Single graph identity:** Equivalent dependency graphs for the - same physical source tree must converge on one runtime dependency instance - within one composed runtime graph. - Refines: DMP-R09, DMP-R11. +- **DMP.LIVE-R05 Single graph identity:** Within one composed runtime graph, + equivalent Dependency Edges for the same physical source must target one + Package Instance. + Refines: DMP-R11. - **DMP.LIVE-R06 Local source linkage:** Workspace and cross-repo local dependencies must resolve to live source when that is the selected topology. Refines: DMP-R11. -- **DMP.LIVE-R07 Dependency truthfulness:** The linker/projection model must not - silently make undeclared dependencies valid. - Refines: DMP-R15, DMP-R16. +- **DMP.LIVE-R07 Dependency truthfulness:** Every realized Dependency Edge must + target the Package Instance selected by pnpm, including its resolved version + and peer context. Projection must not add, remove, or retarget those edges. + Repair must discard the Materialization Root's derived dependency state and + reinvoke pnpm rather than link a replacement. + Refines: DMP-R11, DMP-R15. ### Must remain pure and repairable @@ -70,12 +78,24 @@ dependency graph. lifecycle policy and reject re-enabling dependency scripts. Refines: DMP-R01, DMP-R02, DMP-R03. - **DMP.LIVE-R09 Projection validation:** A skipped or cached install is valid - only when dependency data and projection state are healthy for the selected - topology. + only when Dependency Data and Projection State are healthy for the selected + topology, generated install contract, and declared inputs. A topology- + containment oracle must reject a pnpm Package Instance dependency edge that + escapes its Materialization Root. Exact edge identity remains pnpm's + responsibility under DMP.LIVE-R07; no secondary writer may retarget it. Refines: DMP-R06, DMP-R15. - **DMP.LIVE-R10 Safe concurrency:** CI and disposable task roots must keep - writable pnpm state job-local unless a stronger shared-state proof exists. + their Store Cache job-local. Mutually trusted local development roots may + share a synchronized host-scoped Store Cache; virtual topology and graph + metadata must remain root-local. Refines: DMP-R12, DMP-R13, DMP-R14. - **DMP.LIVE-R11 Observable reuse:** Install reuse, invalidation, repair, and projection decisions must emit machine-readable evidence. Refines: DMP-R19. +- **DMP.LIVE-R12 Mutation parity:** Every managed pnpm entrypoint that installs, + updates, or deduplicates an authoritative lockfile must preserve the same + declared realization policy and authority boundaries. Lockfile mutation must + not silently select a second topology, reuse boundary, lifecycle policy, or + concurrency model, and disposable historical cache state must never become a + correctness or availability dependency. + Refines: DMP-R02, DMP-R11, DMP-R12. diff --git a/context/dependency-materialization/01-live-pnpm/spec.md b/context/dependency-materialization/01-live-pnpm/spec.md index 73f0db4541..0c3d6aad10 100644 --- a/context/dependency-materialization/01-live-pnpm/spec.md +++ b/context/dependency-materialization/01-live-pnpm/spec.md @@ -12,8 +12,8 @@ This spec defines: - live workspace topology authority; - managed install ownership; - mutable state boundaries for local development and CI; -- the live install relation to pure projection, store traits, and profile - evidence. +- the live install relation to pure projection, root-owned state, shared + content, and install evidence. This spec does not define Nix prepared dependency artifacts. Those are specified in [../03-nix-prepared-deps/spec.md](../03-nix-prepared-deps/spec.md). @@ -27,6 +27,7 @@ in [../03-nix-prepared-deps/spec.md](../03-nix-prepared-deps/spec.md). | Runtime Identity | DMP.LIVE-R05, DMP.LIVE-R06, DMP.LIVE-R07 | | CI State | DMP.LIVE-R10 | | Health | DMP.LIVE-R09, DMP.LIVE-R11 | +| Mutation Parity | DMP.LIVE-R12 | ## Model @@ -35,7 +36,7 @@ selected workspace topology -> managed pnpm install with strict policy -> dependency data in live node_modules -> pure projection repair - -> profile evidence and health report + -> install evidence and health report ``` The selected topology, not the current working directory, owns live install @@ -49,7 +50,6 @@ Managed install entrypoints must compute or receive: - `topologyKind`: `standalone`, `composed`, or `packageClosure`; - `lockfile`: the authoritative lockfile for that owner; - `workspaceFile`: the workspace membership file for that owner; -- `profileId`: the dependency materialization profile id. Package-directory installs are not supported when they would create package-local lockfiles, package-local `node_modules`, or a second dependency @@ -68,6 +68,25 @@ repos/effect-utils The parent may link source from a nested root, but it must not silently repair or mutate the nested root's dependency state. +## Mutation Parity + +Every managed entrypoint that can mutate an authoritative lockfile or graph +uses the same realization transaction: + +```text +Materialization-Root lock + package-manager-home lock + -> selected fresh Store Cache namespace + -> shared Store Cache admission lease + -> capacity gate + -> canonical policy arguments + -> pnpm mutation + -> root-local projection +``` + +Install, update, and deduplicate operations cannot select separate topology, +lifecycle, cache, or concurrency policies. Historical cache namespaces remain +outside the transaction and cannot block current materialization. + ## Runtime Identity The live model must preserve one runtime dependency graph for a selected @@ -76,27 +95,51 @@ runtime entrypoints must preserve the logical topology paths required by the runtime so linked packages resolve shared dependencies through the selected graph. -The implementation may use pnpm GVS, hoisting, or future store traits as the -path-collapsing primitive, but the profile must declare that trait and the -doctor must validate it. +Runtime identity is established by the selected standalone or composed +workspace topology. Sharing storage is not an identity primitive: every live +root owns `node_modules/.pnpm`, while mutually trusted roots may reuse one +pnpm-owned Store Cache. + +### Dependency edge selection authority + +pnpm owns dependency-edge selection and realization inside live `node_modules` +and the root-local virtual store. Managed repair discards that root-owned graph, +then asks the root's canonical pnpm install to select and materialize it again. +It does not sweep the host Store Cache and never selects or links a replacement +target itself. + +Missing dependency edges are corrected at a declared authority boundary: + +- the consuming package manifest for real runtime dependencies; +- pnpm `packageExtensions` for a declared package-manager compatibility + extension; +- the generated workspace topology for local source membership. + +This distinction is especially important for peer-dependent packages. Two +store entries with the same package name and version may still represent +different package-instance identities because of peer context, patches, +injected workspace copies, or platform selection. A filesystem repair that +ignores that identity can override a correct pnpm edge with an incompatible +dependency. ## CI State -CI jobs use job-local writable pnpm home, store metadata, and projection state -by default. A shared cache may seed content, but the job remains the mutation -owner unless a stronger shared-state profile has explicit GC and repair -authority. +CI jobs use a job-local pnpm home, Store Cache, virtual topology, and projection +state. The job remains the sole mutation owner for all of those paths. ## Health -A live profile is healthy only when: +A live Materialization Root is healthy only when: -1. the selected topology inputs match the profile evidence; -2. pnpm metadata exists for the declared store trait; +1. the selected topology inputs match the generated install contract and + cached state; +2. root-owned pnpm metadata exists; 3. dependency data is present; -4. expected pure projections exist; -5. offline or no-network usability checks pass for the selected profile when - the store trait promises offline reuse. +4. expected pure projections exist. + +Root health does not imply that a Store Cache contains every package +needed for a future offline reinstall. Offline readiness is a separate claim +that requires its own no-network evidence for the declared inputs. Exit-code downgrades such as pnpm teardown exits are allowed only after these checks prove materialization succeeded. diff --git a/context/dependency-materialization/02-projections/requirements.md b/context/dependency-materialization/02-projections/requirements.md index 18a7fd42f3..1efaf7de12 100644 --- a/context/dependency-materialization/02-projections/requirements.md +++ b/context/dependency-materialization/02-projections/requirements.md @@ -2,17 +2,17 @@ ## Context -Projection is deterministic state derived after dependency data exists. It -includes `node_modules/.bin` entries, workspace package links, and local -metadata needed for tools to execute against a realized dependency graph. +Projection State is deterministic state derived after Dependency Data exists. +It includes `node_modules/.bin` entries and local metadata needed for tools to +execute against a realized Dependency Graph. Projection refines DMP-R05 through DMP-R08. Prepared dependency artifacts are data; projections are recreated and checked by effect-utils-managed steps. ## Assumptions -- **A01 Data exists first:** Projection never resolves dependencies. It operates - on dependency data already materialized by live pnpm or restored from Nix. +- **A01 Graph exists first:** The Authoritative Materializer has completed the + Dependency Graph before projection starts. - **A02 No package code:** Projection reads package metadata and filesystem state but does not execute package code or lifecycle scripts. @@ -27,8 +27,8 @@ data; projections are recreated and checked by effect-utils-managed steps. ### Must own executable projection -- **DMP.PROJ-R01 Bin ownership:** `node_modules/.bin` is profile-owned - projection state, not dependency data. +- **DMP.PROJ-R01 Bin ownership:** `node_modules/.bin` is Projection State owned + by the Materialization Root, not Dependency Data. Refines: DMP-R06. - **DMP.PROJ-R02 Manifest source:** Expected bins must be derived from package manifests and realized package roots. @@ -38,16 +38,16 @@ data; projections are recreated and checked by effect-utils-managed steps. build approval paths. Refines: DMP-R01, DMP-R03, DMP-R17. - **DMP.PROJ-R04 Target validation:** A bin entry may be created only when its - target file exists in dependency data or an explicit Nix/native integration. + target file exists in Dependency Data or an explicit Nix/native integration. Refines: DMP-R04, DMP-R07. ### Must be deterministic and diagnosable -- **DMP.PROJ-R05 Stable output:** The same dependency data and projection policy - must produce the same projection files. +- **DMP.PROJ-R05 Stable output:** The same Dependency Graph, Dependency Data, + and projection policy must produce the same projection files. Refines: DMP-R07, DMP-R15. -- **DMP.PROJ-R06 Owned overwrite:** Stale projection files owned by the profile - must be repaired deterministically. +- **DMP.PROJ-R06 Owned overwrite:** Stale Projection State owned by the + Materialization Root must be repaired deterministically. Refines: DMP-R15. - **DMP.PROJ-R07 Report:** Projection must emit a report that doctor, repair, and benchmarks can consume. @@ -55,3 +55,7 @@ data; projections are recreated and checked by effect-utils-managed steps. - **DMP.PROJ-R08 Prepared-deps exclusion:** Prepared dependency FOD validation must reject archived `.bin` projections by default. Refines: DMP-R05, DMP-R06, DMP-R18. +- **DMP.PROJ-R09 Dependency-edge non-authority:** Projection must not create, + remove, or retarget Dependency Edges. It may read the realized Dependency + Graph only to derive Projection State. + Refines: DMP-R11, DMP-R15. diff --git a/context/dependency-materialization/02-projections/spec.md b/context/dependency-materialization/02-projections/spec.md index c0aabac6d2..a448bec178 100644 --- a/context/dependency-materialization/02-projections/spec.md +++ b/context/dependency-materialization/02-projections/spec.md @@ -22,6 +22,7 @@ This spec defines deterministic projection after dependency data exists: | Report Shape | DMP.PROJ-R07 | | Edge Cases | DMP.PROJ-R02, DMP.PROJ-R04, DMP.PROJ-R05 | | Prepared-deps exclusion | DMP.PROJ-R08 | +| Dependency authority | DMP.PROJ-R09 | ## Bin Projection @@ -44,7 +45,7 @@ realized package root -> resolve declared bins -> validate target is inside package root and exists -> select conflict winner - -> create profile-owned .bin entry + -> create .bin entry owned by Materialization Root ``` The projector: @@ -62,6 +63,12 @@ The projector: The projector does not import package modules, execute package scripts, or call pnpm build commands. +Package dependency edges are outside this projector's authority. It must not +create, remove, or retarget `node_modules/` links. It may read the +realized graph only to derive projector-owned outputs such as `.bin` entries and +reports. The owning materializer repairs an unhealthy dependency graph before +projection runs. + ## Conformance Oracle The conformance fixture compares effect-utils output with pnpm's published bin diff --git a/context/dependency-materialization/03-nix-prepared-deps/01-fod-hash-evidence/requirements.md b/context/dependency-materialization/03-nix-prepared-deps/01-fod-hash-evidence/requirements.md index 02f4ed9934..a186000dbb 100644 --- a/context/dependency-materialization/03-nix-prepared-deps/01-fod-hash-evidence/requirements.md +++ b/context/dependency-materialization/03-nix-prepared-deps/01-fod-hash-evidence/requirements.md @@ -45,8 +45,8 @@ into package source. - **DMP.NIX.FOD-R05 Direct attr:** Evidence must name the direct prepared deps attr to rebuild. Refines: DMP.NIX-R09. -- **DMP.NIX.FOD-R06 Inputs digest:** Evidence must include the prepared profile - id and policy/artifact version digests. +- **DMP.NIX.FOD-R06 Inputs digest:** Evidence must include the prepared + Materialization Profile identity and policy/artifact version digests. Refines: DMP.NIX-R08. - **DMP.NIX.FOD-R07 Diagnostic text:** Stale hash failures must point at the direct dependency artifact, not only the final package. diff --git a/context/dependency-materialization/03-nix-prepared-deps/02-native-node-packages/requirements.md b/context/dependency-materialization/03-nix-prepared-deps/02-native-node-packages/requirements.md index 599f873a5e..516f68f746 100644 --- a/context/dependency-materialization/03-nix-prepared-deps/02-native-node-packages/requirements.md +++ b/context/dependency-materialization/03-nix-prepared-deps/02-native-node-packages/requirements.md @@ -27,7 +27,7 @@ of lifecycle scripts. Refines: DMP-R04. - **DMP.NIX.NATIVE-R03 Pure artifact exception:** A platform package may remain in dependency data only when classified as pure package data for that - profile. + Materialization Profile. Refines: DMP-R04, DMP-R08. - **DMP.NIX.NATIVE-R04 No optional smuggling:** Optional dependencies must not smuggle platform-native outputs into platform-neutral prepared artifacts. diff --git a/context/dependency-materialization/03-nix-prepared-deps/requirements.md b/context/dependency-materialization/03-nix-prepared-deps/requirements.md index 85ba0bafb4..545ccef3c5 100644 --- a/context/dependency-materialization/03-nix-prepared-deps/requirements.md +++ b/context/dependency-materialization/03-nix-prepared-deps/requirements.md @@ -55,15 +55,16 @@ projection semantics from direct dependency boundary consumed by downstream Nix builds. Refines: DMP-R09, DMP-R10. - **DMP.NIX-R06 Per install root:** Composed workspaces must preserve one - prepared dependency boundary per authoritative install root unless a broader - shared profile is explicitly measured and accepted. + prepared dependency boundary per Materialization Root unless a single broader + Materialization Root and Materialization Profile are explicitly measured and + accepted. Refines: DMP-R09, DMP-R11, DMP-R16. - **DMP.NIX-R07 Restore without install:** Downstream builds must restore the prepared artifact and run projection/build steps without rerunning pnpm dependency materialization. Refines: DMP-R05, DMP-R15. -- **DMP.NIX-R08 Evidence:** Each prepared artifact must emit profile evidence, - purity-scan results, and hash-measurement metadata. +- **DMP.NIX-R08 Evidence:** Each prepared artifact must emit Materialization + Profile evidence, purity-scan results, and hash-measurement metadata. Refines: DMP-R10, DMP-R18, DMP-R19. ### Must remain operational diff --git a/context/dependency-materialization/04-store-authority/.experiments/2026-06-22-split-files-pool-prune.md b/context/dependency-materialization/04-store-authority/.experiments/2026-06-22-split-files-pool-prune.md new file mode 100644 index 0000000000..5d4fe1a5d2 --- /dev/null +++ b/context/dependency-materialization/04-store-authority/.experiments/2026-06-22-split-files-pool-prune.md @@ -0,0 +1,35 @@ +# 2026-06-22 Split Files Pool Prune + +## Question + +Can two pnpm stores safely use independent mutable indexes over one shared +`v11/files` pool while each store retains native prune authority? + +## Method + +- Created sibling stores with separate metadata and one shared package-files + pool. +- Installed overlapping dependency graphs. +- Ran raw profile-local prune through only one store. +- Checked the sibling store and attempted offline reinstall. + +## Result + +The profile-local prune removed content still required by the sibling whose +metadata was invisible to the pruning store. Store status could appear clean +while the sibling offline reinstall failed. + +## Conclusion + +Mutually invisible writable indexes cannot each own destructive GC over one +package-files pool. That realization requires an independent complete root-set +GC authority or must remain append-only/fail closed. The finding falsifies the +historical split-`v11/files` implementation; it does not make a shared mutable +whole-store index satisfy the pure reuse boundary. + +## VRS Impact + +- Rejects a direct return to split `v11/files` under DMP.STORE-R03 and + DMP.STORE-R15. +- Constrains DELTA-001 resolution to an immutable seed/artifact or a GC design + that can prove complete reachability without shared mutable consumer state. diff --git a/context/dependency-materialization/04-store-authority/.experiments/2026-06-22-synthetic-store-footprint.md b/context/dependency-materialization/04-store-authority/.experiments/2026-06-22-synthetic-store-footprint.md new file mode 100644 index 0000000000..ae89f645f5 --- /dev/null +++ b/context/dependency-materialization/04-store-authority/.experiments/2026-06-22-synthetic-store-footprint.md @@ -0,0 +1,26 @@ +# 2026-06-22 Synthetic Store Footprint + +## Question + +Does reusing package content reduce host-wide bytes and files relative to +isolated stores? + +## Method + +Materialized synthetic overlapping dependency graphs with isolated and shared +package-content realizations and compared aggregate footprint. + +## Result + +Shared package content produced a material byte and file-count reduction. + +## Conclusion + +The result supports broad reuse of immutable package data as a design signal, +but it does not select a default without integrated real-workspace correctness, +purity, concurrency, repair, and performance gates. + +## VRS Impact + +Supports DMP.STORE-R03 and DMP.STORE-R14 while leaving the realization choice to +the full verification matrix. diff --git a/context/dependency-materialization/04-store-authority/experiments.md b/context/dependency-materialization/04-store-authority/experiments.md deleted file mode 100644 index 39e1319f96..0000000000 --- a/context/dependency-materialization/04-store-authority/experiments.md +++ /dev/null @@ -1,37 +0,0 @@ -# Store Authority Experiments - -This file records non-normative store authority evidence. - -## Split CAS Prune - -Hypothesis: - -- Two pnpm stores that share one package files pool are unsafe if one store runs - raw prune using only its own metadata authority. - -Result: - -- Accepted. A profile-local prune can remove content needed by a sibling - profile whose metadata is invisible to the pruning store. - -Conclusion: - -- Shared content pools need root-set GC authority, and raw profile-local prune - must fail closed. - -## Synthetic Store Trait Footprint - -Hypothesis: - -- Shared content traits reduce host-wide bytes and file counts versus isolated - stores. - -Result: - -- Accepted as a design signal. Synthetic overlapping graphs showed a material - footprint win for shared package content. - -Conclusion: - -- Preserve a sharing trait, but require integrated real-workspace benchmarks - before changing defaults. diff --git a/context/dependency-materialization/04-store-authority/open-questions.md b/context/dependency-materialization/04-store-authority/open-questions.md new file mode 100644 index 0000000000..fe2285943e --- /dev/null +++ b/context/dependency-materialization/04-store-authority/open-questions.md @@ -0,0 +1,13 @@ +# Store Authority Open Questions + +## DQ1: Can reusable topology satisfy the purity and authority gates? + +- Blocks: graduating repeated topology work from root-local pnpm realization + into a broadly reusable Hermetic Dependency Artifact. +- Resolution signal: complete the four-way real-workload comparison in + DMP.VER-R12 on ext4 and APFS, including fault injection and one-root repair. +- Blocker: current committed evidence compares Store Cache sharing strategies, + not topology-sharing strategies. +- Lean: retain root-local pnpm topology as the current compatibility boundary; + use GVS measurements to design an immutable graph-addressed artifact, not to + justify wider shared mutation. diff --git a/context/dependency-materialization/04-store-authority/requirements.md b/context/dependency-materialization/04-store-authority/requirements.md index 3b986fe501..028829c253 100644 --- a/context/dependency-materialization/04-store-authority/requirements.md +++ b/context/dependency-materialization/04-store-authority/requirements.md @@ -2,58 +2,121 @@ ## Context -Store authority defines when dependency content may be shared and who may +Store authority defines which dependency state may be shared and who may repair, prune, or garbage-collect it. It refines DMP-R12 through DMP-R15. ## Assumptions -- **A01 pnpm mutable state:** pnpm metadata, links, side-effects cache, and - projection files are mutable local state unless a Nix artifact proves - otherwise. -- **A02 Sharing is valuable:** Host-wide package-content sharing is a desired - optimization when correctness and repair semantics are explicit. +- **A01 pnpm Store Cache:** A pnpm Store Cache contains immutable + content-addressed package files and pnpm-owned mutable derived indexes. Both + are disposable cache state; neither is Dependency Graph authority. + The mutable indexes do not satisfy the pure cross-root reuse boundary. +- **A02 Local trust boundary:** Local development roots that share a Store + Cache run as one mutually trusted operating-system user. +- **A03 Safety-gated reuse objective:** Cross-worktree reuse is optimized only + after dependency identity, purity, data safety, graph ownership, concurrency, + and bounded repair scope are satisfied as hard constraints. ## Acceptable Tradeoffs -- **T01 Refuse unsafe repair:** Commands may fail closed when they cannot prove - authority over every active root. -- **T02 Trait-specific defaults:** Darwin, Linux, CI, and Nix may use different - store traits under one profile vocabulary. +- **T01 Fail-closed zero-copy:** A local install may refuse to materialize when + its selected zero-copy import method cannot be honored safely. +- **T02 CI isolation:** CI may give up host-wide cache reuse to keep concurrent + jobs and their cleanup independent. +- **T03 Independent Nix path:** Live-install cache placement need not match Nix + prepared-dependency storage because Nix provides its own content-addressed + reuse boundary. +- **T04 Same-user hardlink aliasing:** On a filesystem where pnpm `auto` + selects hardlinks, a direct write through one imported dependency aliases the + Store Cache and sibling roots. This is accepted only inside the declared + mutually trusted same-user boundary; managed installs must keep lifecycle + mutation disabled and must preserve build-sensitive package isolation. ## Requirements ### Must declare authority -- **DMP.STORE-R01 One trait:** Every profile must declare exactly one store - trait. +- **DMP.STORE-R01 Independent storage facts:** Evidence and configuration must + state graph scope, Store Cache scope, import method, and system applicability + as independent facts rather than one preset or profile name. Refines: DMP-R12. -- **DMP.STORE-R02 Mutable owner:** Writable package-manager metadata and - projection state must have one owner. +- **DMP.STORE-R02 Root-local graph owner:** Writable Dependency Graph state, + virtual topology, and Projection State must be owned by exactly one + Materialization Root and independently discardable without coordinating + sibling roots. A storage-reuse mechanism must not expand that authority scope. Refines: DMP-R12. -- **DMP.STORE-R03 Shared pool root set:** A shared content pool may be swept - only by an authority that can enumerate every active root. - Refines: DMP-R13. -- **DMP.STORE-R04 Raw prune refusal:** Profile-local prune must refuse when it - would sweep a shared content pool without root-set authority. +- **DMP.STORE-R03 Maximal safe cache reuse:** Local development must maximize + cross-root reuse of eligible immutable package data within one declared trust + boundary. Mutable package-manager indexes must remain outside the claimed + reusable layer. Duplicate immutable-data realizations require measured + justification and must not become graph, lifecycle, repair, or availability + authority. + Refines: DMP-R12, DMP-R13, DMP-R21, DMP-R22, DMP-R23. +- **DMP.STORE-R04 Root operation boundary:** An effect-utils-managed operation + scoped to one Materialization Root must not prune or sweep a host-scoped + Store Cache. Refines: DMP-R14. -### Must be repairable +### Must be safe and repairable -- **DMP.STORE-R05 Missing content detection:** Health checks must detect - missing shared content needed for offline reuse or projection. - Refines: DMP-R13, DMP-R15. -- **DMP.STORE-R06 Deterministic repair:** Repair must rebuild from declared - inputs and must not rewrite lockfiles. - Refines: DMP-R15. -- **DMP.STORE-R07 Low-disk safety:** Low-disk refusal and recovery must be - explicit rather than leaving a profile apparently healthy but unusable. +- **DMP.STORE-R05 Explicit offline readiness:** Materialization Root health must + not imply Store Cache completeness. Any offline-readiness claim must name its + declared inputs and carry separate no-network evidence. + Refines: DMP-R13, DMP-R19. +- **DMP.STORE-R06 Deterministic root repair:** Repair must rebuild from declared + inputs without rewriting lockfiles, mutating sibling roots, or pruning the + Store Cache. + Refines: DMP-R14, DMP-R15. +- **DMP.STORE-R07 Low-disk safety:** Managed materialization must check its + writable storage boundary before creating a second dependency realization + and must fail explicitly when the configured free-space floor is not met. Refines: DMP-R15. +- **DMP.STORE-R08 Concurrent cache mutation:** Concurrent managed installs that + share a Store Cache must retain independent root-local locks and graphs and + must not serialize independent roots without evidence that the cache owner's + native concurrency boundary is insufficient. + Refines: DMP-R12, DMP-R15. +- **DMP.STORE-R09 Capability-optimal import:** Managed live installs must select + the most reuse-efficient import mechanism proven compatible with the host + filesystem and purity boundary. Imported package files are immutable by + contract; verification must disclose effective byte/inode aliasing rather + than claiming unconditional physical isolation. + Refines: DMP-R01, DMP-R12, DMP-R21, DMP-R22. +- **DMP.STORE-R10 Zero-copy invariant:** A local install that claims zero-copy + reuse must prove its placement and filesystem prerequisites before mutation + and fail closed instead of silently degrading to per-root byte duplication. + Refines: DMP-R12, DMP-R22. +- **DMP.STORE-R11 CI job cache:** CI installs must use a job-local Store Cache; + one job's cleanup must not mutate another job's + cache or graph. + Refines: DMP-R12, DMP-R13. +- **DMP.STORE-R12 Independent Nix cache:** Nix prepared-dependency production + must use its builder-owned immutable/content-addressed boundary rather than + mutable live host cache state. + Refines: DMP-R05, DMP-R09, DMP-R21, DMP-R23. ### Must be measured -- **DMP.STORE-R08 Benchmark matrix:** Candidate traits must report cold, warm, - offline, concurrent, byte, and file-count metrics. - Refines: DMP-R16. -- **DMP.STORE-R09 Default gate:** A trait may become default only after proving - correctness and material cache-efficiency gains on real workspaces. +- **DMP.STORE-R13 Benchmark evidence:** Changes to storage placement, sharing, + or import method must report the comparison evidence defined by the + verification subsystem. Refines: DMP-R16. +- **DMP.STORE-R14 Default gate:** A sharing or import strategy may become + default only after satisfying the hard gates in DMP-R21 through DMP-R23 and + proving a non-dominated operating point among evaluated admissible candidates + on real workspaces across physical bytes, repeated work, latency, concurrency, + and operational complexity. New admissible candidates remain challengers. + Refines: DMP-R16, DMP-R21, DMP-R22, DMP-R23. +- **DMP.STORE-R15 Bounded host lifecycle:** The host Store Cache owner must + measure cache bytes periodically and under pressure. Destructive reclamation + may be enabled only when evidence proves it cannot invalidate live roots; + otherwise it remains measurement-only. Every run must report cache bytes, + reclaimed bytes, outcome, and dry-run mode. Maintenance must exclude cache + mutation without serializing independent installs against one another. + Refines: DMP-R13, DMP-R14, DMP-R15. +- **DMP.STORE-R16 Explicit legacy-cache migration:** A legacy Store Cache that + bridges package data outside its selected ownership boundary must fail closed + during normal installs. Only an explicit, idempotent cache-owner migration may + transform a recognized legacy shape; unknown state and the external legacy + data source must remain untouched. + Refines: DMP-R13, DMP-R14, DMP-R15. diff --git a/context/dependency-materialization/04-store-authority/spec.md b/context/dependency-materialization/04-store-authority/spec.md index 3c1310cb54..a46cf2a794 100644 --- a/context/dependency-materialization/04-store-authority/spec.md +++ b/context/dependency-materialization/04-store-authority/spec.md @@ -5,48 +5,151 @@ This document specifies store authority. It builds on Status: **Draft** +## Scope + +This spec defines steady-state storage placement, graph isolation, package +import selection, and root-repair boundaries. It does not define transitional +legacy-store migration, a root registry, named storage profiles, all-root +repair, or machine-specific Store Cache placement. + ## Requirement Trace -| Section | Requirements | -| ------------------ | ------------------------------------------- | -| Traits | DMP.STORE-R01, DMP.STORE-R02 | -| Shared Pool GC | DMP.STORE-R03, DMP.STORE-R04 | -| Health And Repair | DMP.STORE-R05, DMP.STORE-R06, DMP.STORE-R07 | -| Benchmark evidence | DMP.STORE-R08, DMP.STORE-R09 | +| Section | Requirements | +| ------------------ | ---------------------------------------------------------- | +| Ownership model | DMP.STORE-R01, DMP.STORE-R02, DMP.STORE-R03 | +| Placement | DMP.STORE-R03, DMP.STORE-R11, DMP.STORE-R12 | +| Import policy | DMP.STORE-R09, DMP.STORE-R10 | +| Concurrency | DMP.STORE-R08 | +| Health and repair | DMP.STORE-R04, DMP.STORE-R05, DMP.STORE-R06, DMP.STORE-R07 | +| Benchmark evidence | DMP.STORE-R13, DMP.STORE-R14 | +| Host lifecycle | DMP.STORE-R15 | +| Legacy migration | DMP.STORE-R16 | +| Optimization order | DMP.STORE-R03, DMP.STORE-R09, DMP.STORE-R14 | + +## Ownership Model + +```text +same-user local Materialization Roots + root A: node_modules/.pnpm + graph + projection + root B: node_modules/.pnpm + graph + projection + | + +--> host pnpm Store Cache + - immutable content-addressed files + - pnpm-owned mutable derived index + +CI job + root-local graph + job-local Store Cache + +Nix builder + prepared dependencies + builder-owned store +``` + +The pnpm Store Cache is a performance cache, not a dependency-identity or +availability authority. The current compatibility realization shares its +mutable index inside the same-user trust boundary under pnpm synchronization, +but that index is not part of the pure reusable layer and remains an explicit +[implementation delta](../.delta/DELTA-001-whole-store-mutable-index.md). +Dependency edges and peer-context topology remain in each root's +`node_modules/.pnpm` with `enable-global-virtual-store=false`. + +This is the current admissible pnpm baseline, not the long-term reuse ideal. +Current GVS realizations share mutable topology and repair state and therefore +fail the pure reuse and bounded-authority gates even if they save bytes or time. +The verification contract still measures them to quantify repeated topology +work and inform a future immutable, graph-addressed dependency artifact. + +## Placement -## Traits +| Realization | Store Cache scope | Virtual-store scope | Cleanup authority | +| ----------------- | ----------------- | -------------------- | ----------------- | +| local development | one host/user | Materialization Root | host cache owner | +| CI | one job | Materialization Root | that CI job | +| Nix prepared deps | builder-owned | builder-owned | Nix | -| Trait | Use | Writable state | Shared content | GC authority | -| --------------------- | ----------------------------------- | --------------------------------- | ----------------------- | ----------------------- | -| `ciJobLocal` | CI/disposable tasks | job-local | none | profile | -| `darwinSplitCas` | macOS local development | profile-local metadata/projection | shared pnpm files pool | shared-pool coordinator | -| `linuxSharedHardlink` | Linux local development after proof | host-local metadata | hardlink-friendly store | shared-pool coordinator | -| `isolated` | fallback/debug | profile-local | none | profile | -| `nixPreparedDeps` | Nix prepared data | immutable Nix store output | Nix store | Nix store | -| `frozenSeed` | future seed | writable overlay if proven | immutable seed | seed owner plus profile | +The local Store Cache path is configurable so operators can place it on an +appropriate volume. Storage placement is excluded from Materialization Profile +identity: moving or discarding a cache does not change dependency identity. -## Shared Pool GC +## Import Policy + +| Context | Import method | Gate | +| ----------------- | ------------- | ----------------------------------------- | +| Linux local dev | `auto` | cache files and root have equal device ID | +| Darwin local dev | `auto` | pnpm filesystem-capability selection | +| CI | `auto` | job-local Store Cache | +| Nix prepared deps | Nix policy | independent of live-install policy | + +Linux fails before installation when device IDs differ; it does not silently +copy and turn a zero-copy goal into per-worktree duplication. On a filesystem +without clone support, pnpm's native `auto` policy may select hardlinks inside +the explicitly mutually trusted same-user boundary. Managed installs never run +package lifecycle mutation over imported dependency files. + +The managed contract treats imported dependency files as immutable even when +the operating-system user could deliberately change permissions and mutate a +hardlinked inode. Such direct mutation is outside the trust boundary, while +integrity checks and rematerialization detect or replace corrupted data. + +## Concurrency + +pnpm owns concurrency inside its Store Cache. Materialization-root locks remain +independent and protect each root's graph and projection. A shared Store Cache +admission lease composes pnpm mutation with host maintenance: every managed +install takes a shared lease, while pruning takes its exclusive counterpart. +Shared leases are compatible, so this is not a host-wide install mutex. Every +managed graph mutation (`install`, lockfile update, or deduplication) therefore +composes as: ```text -active profiles - -> enumerate referenced metadata roots - -> mark referenced package content - -> sweep only unmarked content +Materialization-Root lock + package-manager-home lock + -> shared Store Cache admission lease + -> capacity gate + -> pnpm graph mutation under one realization policy + -> root-local projection ``` -A command that can see only one profile's metadata must not sweep a shared -content pool. +No registry of roots is required: the Store Cache is disposable and does not +own graph identity. + +## Host Lifecycle + +The host cache owner periodically measures the whole pnpm Store Cache and may +trigger pnpm-native `store prune` on schedule or under disk pressure only +after host-specific evidence proves that the effective import method exposes +live-root reachability to pnpm's pruning semantics. Measured hardlink hosts may +enable destructive pruning; clone, copy, CoW, and unproven hosts remain +measurement-only. Maintenance takes the exclusive Store Cache lease, then +reports bytes before, bytes after, reclaimed bytes, outcome, and dry-run mode. +Root-scoped repair never invokes this operation. + +The cache owner does not enumerate Materialization Roots and never deletes +individual pnpm internals itself. Eviction may make a future offline install +miss; it cannot change a declared graph or an already-materialized root. + +Legacy external `v11/files` bridges are not followed by managed installs. The +explicit `pnpm:store:migrate-legacy` operation takes the exclusive maintenance +lease, accepts only the declared historical files-pool target, and resets the +disposable v11 metadata inside the selected Store Cache. It preserves the store +root and maintenance-lock inode and leaves the external historical pool +untouched. An already self-contained cache is a successful no-op; any unknown +bridge fails closed. ## Health And Repair Store health checks verify: -- profile evidence matches selected inputs; -- metadata exists for the selected trait; -- shared package content referenced by the profile is present; +- declared inputs and root install evidence agree; +- root-owned graph metadata exists; +- dependency data referenced by the current graph is present; - projection health checks pass; -- offline/no-network checks pass when promised by the trait. +- Linux zero-copy preconditions hold before materialization. + +These checks establish current root health, not Store Cache completeness for a +future reinstall. Offline readiness is separate evidence produced by a +no-network reinstall for named declared inputs. -Repair may rerun strict pure materialization, rebuild projection, or ask the -shared-pool coordinator to repair/GC. Repair may not run lifecycle scripts or -rewrite lockfiles. +Root repair discards only that root's graph and projections, then reruns strict +pure materialization. Repair may not run lifecycle scripts, mutate sibling +roots, sweep the host Store Cache, or rewrite lockfiles. A low-disk gate runs +before a replacement realization is created so repair does not require an +unbounded overlap of old and new roots. diff --git a/context/dependency-materialization/05-buck2-evidence/requirements.md b/context/dependency-materialization/05-buck2-evidence/requirements.md index 579a12aa84..ca616d28aa 100644 --- a/context/dependency-materialization/05-buck2-evidence/requirements.md +++ b/context/dependency-materialization/05-buck2-evidence/requirements.md @@ -8,15 +8,15 @@ dependency building or host-local pnpm repair. ## Assumptions -- **A01 Profile authority:** Buck2 evidence consumes the shared DMP profile - identity. +- **A01 Materialization Profile identity:** Buck2 evidence consumes the shared + DMP Materialization Profile identity. - **A02 No live repair:** Live mutable pnpm install and repair remain outside Buck2 until a hermetic action is proven. ## Acceptable Tradeoffs -- **T01 Evidence first:** Buck2 may start with profile evidence targets instead - of full dependency materialization targets. +- **T01 Evidence first:** Buck2 may start with Materialization Profile evidence + targets instead of full dependency materialization targets. ## Requirements @@ -25,8 +25,8 @@ dependency building or host-local pnpm repair. - **DMP.BUCK-R01 Declared inputs:** Buck2 targets must depend on declared dependency inputs or immutable artifacts, not ambient pnpm store contents. Refines: DMP-R09, DMP-R10. -- **DMP.BUCK-R02 Stable evidence:** Evidence must include profile identity, - policy digest, input digests, and materialization authority. +- **DMP.BUCK-R02 Stable evidence:** Evidence must include Materialization Profile + identity, policy digest, input digests, and materialization authority. Refines: DMP-R10. - **DMP.BUCK-R03 No secret keys:** Evidence must not include credentials or host-private paths. @@ -38,6 +38,6 @@ dependency building or host-local pnpm repair. run live pnpm install, shared-store GC, or repair. Refines: DMP-R11, DMP-R12. - **DMP.BUCK-R05 Future hermetic path:** A future Buck2 dependency builder must - declare the same profile inputs and prove output equivalence against the Nix - or live profile realization it replaces. + declare the same Materialization Profile inputs and prove output equivalence + against the Nix or live realization it replaces. Refines: DMP-R10, DMP-R16. diff --git a/context/dependency-materialization/06-observability/requirements.md b/context/dependency-materialization/06-observability/requirements.md index ede2d05c6a..c92d910758 100644 --- a/context/dependency-materialization/06-observability/requirements.md +++ b/context/dependency-materialization/06-observability/requirements.md @@ -26,8 +26,8 @@ translate them into spans, events, dashboards, or review evidence. - **DMP.OBS-R03 Reuse outcome:** Cache hit, reuse, invalidation, bypass, and repair decisions must be explicit. Refines: DMP-R19. -- **DMP.OBS-R04 Stable profile linkage:** Facts must include profile ids or - safe profile references when available. +- **DMP.OBS-R04 Stable profile linkage:** Facts must include Materialization + Profile identities or safe Materialization Profile references when available. Refines: DMP-R10, DMP-R19. ### Must be machine-readable diff --git a/context/dependency-materialization/07-verification/.delta/DELTA-001-legacy-research-routing.md b/context/dependency-materialization/07-verification/.delta/DELTA-001-legacy-research-routing.md new file mode 100644 index 0000000000..f8f968e0e4 --- /dev/null +++ b/context/dependency-materialization/07-verification/.delta/DELTA-001-legacy-research-routing.md @@ -0,0 +1,41 @@ +# DELTA-001: Imported research uses a parallel companion taxonomy + +Status: open + +## Divergence + +The verification subtree still contains imported snapshots under `.research/`, +which is not a current VRS companion kind. The snapshots preserve useful source +evidence but deterministic strict validation does not route or inspect them. + +## VRS + +- The repository VRS contract permits source-backed external facts under + `.reference/` and project-generated experiments under `.experiments/`. +- [Verification evidence intake](../spec.md) requires imported findings to + graduate into fixtures, focused experiments, benchmarks, pending evidence, or + explicit rejection rather than remain a parallel authority. + +## Implementation + +`07-verification/.research/README.md`, +`downstream-dependency-profile-research.md`, and `proof-catalog.md` retain the +historical imported taxonomy. The focused consolidation experiment records +which conclusions have already graduated or been superseded. + +## Resolution Approach + +Route source snapshots with provenance to `.reference/`; split project-generated +proofs into focused `.experiments/` or executable evidence; then remove the +`.research/` directory and this delta without creating another research ledger. + +## Direction + +update implementation + +## Resolution Signal + +- Every retained source fact has reference provenance or focused executable + evidence. +- No normative decision depends only on `.research/` content. +- The `.research/` directory and this delta are removed. diff --git a/context/dependency-materialization/07-verification/.experiments/2026-06-24-downstream-profile-research-consolidation.md b/context/dependency-materialization/07-verification/.experiments/2026-06-24-downstream-profile-research-consolidation.md index cbe7d3cdee..d7281e25df 100644 --- a/context/dependency-materialization/07-verification/.experiments/2026-06-24-downstream-profile-research-consolidation.md +++ b/context/dependency-materialization/07-verification/.experiments/2026-06-24-downstream-profile-research-consolidation.md @@ -3,13 +3,13 @@ This file records non-normative evidence for dependency materialization verification. Normative behavior lives in [../spec.md](../spec.md). -## Hypothesis +## Question The downstream pnpm/Nix/Buck2 research package can be retired once every durable finding is represented in effect-utils as a VRS requirement, spec rule, fixture/proof, benchmark shape, pending evidence marker, or explicit rejection. -## Source Material +## Method The imported research is preserved in this tree: @@ -20,7 +20,7 @@ The imported research is preserved in this tree: The research package contains durable proof categories: -- split shared-CAS prune failure; +- split shared-files-pool prune failure; - store-status false-clean evidence; - guard, doctor, and repair decision models; - synthetic and real-workload store-trait benchmarks; @@ -48,3 +48,17 @@ The reusable long-term shape belongs in effect-utils as verification requirements, fixtures, proof harnesses, benchmark records, and pending evidence markers. The downstream branch no longer needs to remain a parallel VRS source of truth once its production-relevant findings are represented here. + +The imported registry-backed all-root repair conclusion is superseded for the +whole Store Cache realization. Its split-files-pool corruption proof remains +valid; current repair keeps graphs root-local and delegates whole-cache lifecycle +to the package manager under the cache-owner boundary. + +## VRS Impact + +- The split-pool failure remains negative evidence for DMP.STORE-R03 and + DMP.STORE-R16. +- Current Store Cache lifecycle and repair semantics live in + [04-store-authority](../../04-store-authority/spec.md) and decision 0006. +- Historical named live profiles, raw shared-CAS language, root registries, and + coordinated all-root repair are not current normative architecture. diff --git a/context/dependency-materialization/07-verification/.experiments/README.md b/context/dependency-materialization/07-verification/.experiments/README.md deleted file mode 100644 index 47a91b1935..0000000000 --- a/context/dependency-materialization/07-verification/.experiments/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Dependency Materialization Verification Experiments - -This directory holds non-normative evidence records for dependency -materialization verification. Each record states a hypothesis, source or method, -result, and conclusion. - -Normative behavior lives in [../spec.md](../spec.md). Durable findings from -experiments graduate into requirements, fixtures, reusable proof harnesses, -benchmark records, pending evidence markers, or explicit rejections. diff --git a/context/dependency-materialization/07-verification/.research/downstream-dependency-profile-research.md b/context/dependency-materialization/07-verification/.research/downstream-dependency-profile-research.md index 637135bf61..3e02322399 100644 --- a/context/dependency-materialization/07-verification/.research/downstream-dependency-profile-research.md +++ b/context/dependency-materialization/07-verification/.research/downstream-dependency-profile-research.md @@ -19,7 +19,11 @@ the pnpm/Nix/Buck2 relationship stayed implicit: - Nix prepared dependency freshness could drift from the same dependency inputs used by live installs. -## Design Decisions Captured +## Historical Design Decisions Captured + +The following list records the imported branch's conclusions, not the current +normative architecture. Decision 0006 and the current subsystem specs supersede +its named live profiles, shared-files registry, and coordinated all-root repair. - Nix/devenv remains the owner of live mutable pnpm materialization and repair. - Buck2 consumes declared dependency profile evidence first; it does not run @@ -41,7 +45,7 @@ the pnpm/Nix/Buck2 relationship stayed implicit: | ------------------- | --------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | | Shared-store prune | Profile-local prune can delete files required by sibling roots sharing `v11/files`. | store authority and verification | | Store status limits | `pnpm store status` can report clean after sibling prune even though offline reinstall fails. | store authority health checks | -| Doctor/repair | Correct repair is registry-backed all-root repair, not a thin prune wrapper. | store authority and live pnpm repair | +| Doctor/repair | Historical split-files repair model used registry-backed all-root repair; current whole-cache repair supersedes it. | store authority and live pnpm repair | | Store traits | Shared/split stores preserve large host-wide byte and file-count wins over isolated stores. | verification benchmark matrix | | CI isolation | Job-local pnpm stores avoid sibling corruption and stay the CI default. | store trait contract | | Low disk | Broad proofs must fail before mutation and emit machine-readable skip evidence. | verification skip records | @@ -50,7 +54,11 @@ the pnpm/Nix/Buck2 relationship stayed implicit: | Nix FOD freshness | FOD freshness can use profile identity plus FOD input digest instead of parallel stale-hash heuristics. | Nix prepared deps and FOD evidence | | Buck2 evidence | Buck2 should consume deterministic evidence and keep mutable materialization outside watched source roots. | Buck2 evidence subsystem | -## Open Gaps Preserved +## Historical Open Gaps + +These gaps describe the imported research snapshot. Current status is owned by +the verification requirements, evidence bundle, and store-authority open +questions rather than this reference note. - Linux real-workload numbers were still pending; the Linux runner shape was proven locally by emitting a deterministic non-Linux skip. diff --git a/context/dependency-materialization/07-verification/.research/proof-catalog.md b/context/dependency-materialization/07-verification/.research/proof-catalog.md index 80fbf3af5e..f7d23551cf 100644 --- a/context/dependency-materialization/07-verification/.research/proof-catalog.md +++ b/context/dependency-materialization/07-verification/.research/proof-catalog.md @@ -6,6 +6,11 @@ the current contract. ## Shared Store Authority +The split-files fixtures below preserve historical failure evidence. Their +registry/all-roots repair proposal is superseded by decision 0006: the current +whole Store Cache has one package-manager-owned index, while graph repair stays +inside one Materialization Root. + ### Split store prune hazard Hypothesis: a macOS-style split store with profile-local metadata and shared @@ -77,8 +82,11 @@ Registry model: {"phase":"empty-repair-plan","status":"refuse","reason":"no-registered-roots"} ``` -Conclusion: production repair can be a deterministic registry lookup, -files-pool classification, sibling enumeration, and all-roots repair plan. +Historical conclusion: the explored split-files realization proposed a +registry lookup, files-pool classification, sibling enumeration, and all-roots +repair plan. Current architecture rejects that complexity: it shares the whole +pnpm Store Cache, delegates cache lifecycle to pnpm, and repairs only root-owned +graph/projection state. ## Store-Trait Benchmarks @@ -108,8 +116,10 @@ All copy and hardlink phases passed offline reinstall. Total benchmark tree footprint dropped from 273 MiB with copy import to 135 MiB with hardlink import. -Conclusion: `linuxSharedHardlink` is a first-class candidate trait, but it -still needs real-repo and concurrent-run proof before becoming a default. +Historical conclusion: `linuxSharedHardlink` was a candidate named trait. +Decision 0006 supersedes named live store traits with independent reuse, +authority, import, and platform facts; the current spec uses pnpm `auto` behind +a Linux same-device zero-copy gate. ### Downstream monorepo APFS real profile @@ -130,9 +140,10 @@ Footprint after two cold materializations: Offline reinstall reused 998 packages, downloaded 0 packages, and completed for all traits. -Conclusion: the sharing motivation holds at real graph scale. The correct -direction is coordinated all-roots authority for shared content, not a blanket -fallback to isolated stores. +Conclusion: the sharing motivation holds at real graph scale. The imported +branch proposed coordinated all-roots authority, but current evidence and +decision 0006 preserve the byte-sharing result with a whole Store Cache and +root-local graph authority instead. ### effect-utils APFS real profile diff --git a/context/dependency-materialization/07-verification/evidence/measure-prune-capability.sh b/context/dependency-materialization/07-verification/evidence/measure-prune-capability.sh new file mode 100755 index 0000000000..66fb0672bf --- /dev/null +++ b/context/dependency-materialization/07-verification/evidence/measure-prune-capability.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +set -euo pipefail + +pnpm_bin="$(realpath "${PNPM_BIN:-$(command -v pnpm)}")" +root_base="$(realpath "${ROOT_BASE:-/tmp}")" +store_base="$(realpath "${STORE_BASE:-/tmp}")" +root_tmp="$(mktemp -d "$root_base/.pnpm-prune-capability-root.XXXXXX")" +store_tmp="$(mktemp -d "$store_base/.pnpm-prune-capability-store.XXXXXX")" +trap 'rm -rf "$root_tmp" "$store_tmp"' EXIT +root="$root_tmp/root" +store="$store_tmp/store" +mkdir -p "$root" "$store" + +cat >"$root/package.json" <<'JSON' +{ + "name": "pnpm-prune-capability", + "private": true, + "packageManager": "pnpm@11.8.0", + "dependencies": { + "is-number": "7.0.0" + } +} +JSON +cat >"$root/pnpm-workspace.yaml" <<'YAML' +packages: + - . +YAML + +pnpm_version="$(cd "$root" && "$pnpm_bin" --version)" +if [ -n "${EXPECTED_PNPM_VERSION:-}" ] && [ "$pnpm_version" != "$EXPECTED_PNPM_VERSION" ]; then + printf 'pnpm version mismatch: expected %s, got %s from %s\n' "$EXPECTED_PNPM_VERSION" "$pnpm_version" "$pnpm_bin" >&2 + exit 1 +fi + +( + cd "$root" + "$pnpm_bin" install \ + --ignore-scripts \ + --config.enable-global-virtual-store=false \ + --config.virtual-store-dir=node_modules/.pnpm \ + --config.package-import-method=auto \ + --config.store-dir="$store" \ + --reporter=silent +) + +root_file="$root/node_modules/.pnpm/is-number@7.0.0/node_modules/is-number/index.js" +test -f "$root_file" +root_device="$(stat -c %d "$root_file")" +root_inode="$(stat -c %i "$root_file")" +root_nlink="$(stat -c %h "$root_file")" +store_alias="$(find "$store/v11/files" -type f -inum "$root_inode" -print -quit)" +test -n "$store_alias" +test "$(stat -c %d "$store_alias")" = "$root_device" +test "$root_nlink" -ge 2 + +before_kib="$(du -sk "$store" | awk '{print $1}')" +(cd "$root" && "$pnpm_bin" store prune --config.store-dir="$store" --reporter=silent) +after_live_prune_kib="$(du -sk "$store" | awk '{print $1}')" +test -f "$store_alias" +test "$(node -p "require('$root/node_modules/is-number')(42)")" = true + +rm -rf "$root/node_modules" +(cd "$root" && "$pnpm_bin" store prune --config.store-dir="$store" --reporter=silent) +after_removed_prune_kib="$(du -sk "$store" | awk '{print $1}')" +test ! -e "$store_alias" + +printf '{"schema":"dependency-materialization-verification/v0","kind":"host-capability","surface":"pnpm-store-prune","host":"%s","platform":"%s","filesystem":"%s","pnpm":"%s","pnpmBin":"%s","packageImportMethod":"auto","rootBase":"%s","storeBase":"%s","rootDevice":%s,"storeDevice":%s,"rootInode":%s,"rootNlink":%s,"storeAliasFound":true,"liveRootSurvivedPrune":true,"removedRootCacheEvicted":true,"beforeKiB":%s,"afterLivePruneKiB":%s,"afterRemovedPruneKiB":%s,"destructivePruneSafe":true}\n' \ + "$(hostname -s)" "$(uname -m)-linux" "$(findmnt -n -o FSTYPE --target "$root_base")" "$pnpm_version" "$pnpm_bin" \ + "$root_base" "$store_base" "$root_device" "$(stat -c %d "$store")" "$root_inode" "$root_nlink" "$before_kib" "$after_live_prune_kib" "$after_removed_prune_kib" diff --git a/context/dependency-materialization/07-verification/evidence/measure-storage-sharing-default.sh b/context/dependency-materialization/07-verification/evidence/measure-storage-sharing-default.sh new file mode 100755 index 0000000000..8050bc8e68 --- /dev/null +++ b/context/dependency-materialization/07-verification/evidence/measure-storage-sharing-default.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash +set -euo pipefail + +tmp="$(mktemp -d /tmp/pnpm-store-gate.XXXXXX)" +trap 'rm -rf "$tmp"' EXIT + +if [ -n "${SOURCE_TREE:-}" ]; then + implementation_head="${IMPLEMENTATION_HEAD:?set IMPLEMENTATION_HEAD when SOURCE_TREE is used}" + implementation_tree="${IMPLEMENTATION_TREE:?set IMPLEMENTATION_TREE when SOURCE_TREE is used}" + source_mode=verified-git-archive-tree + test ! -e "$SOURCE_TREE/.git" + cp -R "$SOURCE_TREE" "$tmp/root-a" + cp -R "$SOURCE_TREE" "$tmp/root-b" + cp -R "$SOURCE_TREE" "$tmp/root-c" + cp -R "$SOURCE_TREE" "$tmp/root-d" + cp -R "$SOURCE_TREE" "$tmp/root-e" + git -C "$tmp/root-a" init --quiet + git -C "$tmp/root-a" add --force --all + measured_tree="$(git -C "$tmp/root-a" write-tree)" + rm -rf "$tmp/root-a/.git" + if [ "$measured_tree" != "$implementation_tree" ]; then + printf 'SOURCE_TREE mismatch: expected Git tree %s, measured %s\n' "$implementation_tree" "$measured_tree" >&2 + exit 1 + fi +else + source_repo="${SOURCE_REPO:-$(git rev-parse --show-toplevel)}" + implementation_head="${IMPLEMENTATION_HEAD:-$(git -C "$source_repo" rev-parse HEAD)}" + implementation_tree="$(git -C "$source_repo" rev-parse "$implementation_head^{tree}")" + source_mode=git-worktree + git clone --quiet --no-checkout "$source_repo" "$tmp/repo" + git -C "$tmp/repo" worktree add --quiet --detach "$tmp/root-a" "$implementation_head" + git -C "$tmp/repo" worktree add --quiet --detach "$tmp/root-b" "$implementation_head" + git -C "$tmp/repo" worktree add --quiet --detach "$tmp/root-c" "$implementation_head" + git -C "$tmp/repo" worktree add --quiet --detach "$tmp/root-d" "$implementation_head" + git -C "$tmp/repo" worktree add --quiet --detach "$tmp/root-e" "$implementation_head" +fi + +pnpm_bin="${PNPM_BIN:-$(command -v pnpm)}" +node_bin="$(command -v node)" +store="$tmp/store" +mkdir -p "$store" + +case "$(uname -s):$(uname -m)" in + Linux:x86_64) platform=x86_64-linux ;; + Darwin:arm64) platform=aarch64-darwin ;; + *) platform="$(uname -m)-$(uname -s | tr '[:upper:]' '[:lower:]')" ;; +esac +if [ "$(uname -s)" = Darwin ]; then + filesystem=apfs +else + filesystem="$(findmnt -n -o FSTYPE --target "$tmp" 2>/dev/null || stat -f -c %T "$tmp")" +fi +if command -v sha256sum >/dev/null 2>&1; then + harness_sha256="$(sha256sum "$0" | awk '{print $1}')" +else + harness_sha256="$(shasum -a 256 "$0" | awk '{print $1}')" +fi + +now_ms() { + perl -MTime::HiRes=time -e 'printf "%.0f\n", time * 1000' +} + +measure_tree() { + local label="$1" + local path="$2" + local physical_kib=0 + local apparent_kib=0 + local files=0 + if [ -e "$path" ]; then + physical_kib="$(du -sk "$path" | awk '{print $1}')" + if [ "$(uname -s)" = Darwin ]; then + apparent_kib="$(du -skA "$path" | awk '{print $1}')" + else + apparent_kib="$(du -sk --apparent-size "$path" | awk '{print $1}')" + fi + files="$(find "$path" -type f | wc -l | tr -d ' ')" + fi + printf '{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"%s","phase":"size:%s","status":"ok","sizes":{"physicalKiB":%s,"apparentKiB":%s,"files":%s}}\n' \ + "$platform" "$label" "$physical_kib" "$apparent_kib" "$files" +} + +install_root() { + local phase="$1" + local root="$2" + local active_store="$3" + local start end rc + start="$(now_ms)" + set +e + ( + cd "$root" + CI='' NODE_OPTIONS="${NODE_OPTIONS:+$NODE_OPTIONS }--max-old-space-size=1536" PNPM_SHARED_STORE_DIR="$active_store" "$pnpm_bin" install \ + --frozen-lockfile \ + --ignore-scripts \ + --config.side-effects-cache=false \ + --config.verify-store-integrity=true \ + --config.strict-store-pkg-content-check=true \ + --config.enable-global-virtual-store=false \ + --config.virtual-store-dir=node_modules/.pnpm \ + --config.package-import-method=auto \ + --config.store-dir="$active_store" \ + --child-concurrency=1 \ + --network-concurrency=4 \ + --reporter=append-only + ) >"$tmp/$phase.log" 2>&1 + rc=$? + set -e + if [ "$rc" -eq 134 ] && + [ "$(uname -s)" = Darwin ] && + grep -qE 'Progress: .* done$' "$tmp/$phase.log" && + [ -d "$root/node_modules/.pnpm" ] && + [ -f "$root/node_modules/.modules.yaml" ]; then + completed_materialization=true + rc=0 + else + completed_materialization=false + fi + if [ "$rc" -ne 0 ]; then + printf '{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"%s","phase":"%s","status":"failed","exitCode":%s}\n' "$platform" "$phase" "$rc" + tail -n 100 "$tmp/$phase.log" >&2 + return "$rc" + fi + end="$(now_ms)" + local progress_line reused downloaded teardown_exit + progress_line="$(grep -E 'Progress: .* done$' "$tmp/$phase.log" | tail -n 1)" + reused="$(printf '%s\n' "$progress_line" | sed -nE 's/.*reused ([0-9]+).*/\1/p')" + downloaded="$(printf '%s\n' "$progress_line" | sed -nE 's/.*downloaded ([0-9]+).*/\1/p')" + teardown_exit=0 + if [ "$completed_materialization" = true ]; then + teardown_exit=134 + fi + printf '{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"%s","phase":"%s","status":"ok","durationMs":%s,"reused":%s,"downloads":%s,"teardownExit":%s,"completedMaterializationEvidence":%s}\n' \ + "$platform" "$phase" "$((end - start))" "${reused:-0}" "${downloaded:-0}" "$teardown_exit" "$completed_materialization" + measure_tree "$phase-root-node-modules" "$root/node_modules" + measure_tree "$phase-store" "$active_store" +} + +measure_combined() { + local label="$1" + shift + local physical_kib apparent_kib files + physical_kib="$(du -skc "$@" | tail -n 1 | awk '{print $1}')" + if [ "$(uname -s)" = Darwin ]; then + apparent_kib="$(du -skAc "$@" | tail -n 1 | awk '{print $1}')" + else + apparent_kib="$(du -skc --apparent-size "$@" | tail -n 1 | awk '{print $1}')" + fi + files="$(find "$@" -type f | wc -l | tr -d ' ')" + printf '{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"%s","phase":"size:%s","status":"ok","sizes":{"physicalKiB":%s,"apparentKiB":%s,"files":%s}}\n' \ + "$platform" "$label" "$physical_kib" "$apparent_kib" "$files" +} + +printf '{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"%s","phase":"provenance","status":"ok","implementationHead":"%s","implementationTree":"%s","harnessSha256":"%s","sourceMode":"%s","filesystem":"%s","pnpm":"%s","node":"%s"}\n' \ + "$platform" "$implementation_head" "$implementation_tree" "$harness_sha256" "$source_mode" "$filesystem" "$("$pnpm_bin" --version)" "$("$node_bin" --version)" + +install_root cold-root-a "$tmp/root-a" "$store" +install_root second-root-b "$tmp/root-b" "$store" +measure_combined shared-two-roots "$store" "$tmp/root-a/node_modules" "$tmp/root-b/node_modules" +install_root warm-root-b "$tmp/root-b" "$store" +install_root isolated-root-c "$tmp/root-c" "$tmp/isolated-store" +measure_combined isolated-one-root "$tmp/isolated-store" "$tmp/root-c/node_modules" + +concurrent_start="$(now_ms)" +set +e +install_root concurrent-root-d "$tmp/root-d" "$store" >"$tmp/concurrent-root-d.jsonl" & +concurrent_d_pid=$! +install_root concurrent-root-e "$tmp/root-e" "$store" >"$tmp/concurrent-root-e.jsonl" & +concurrent_e_pid=$! +wait "$concurrent_d_pid" +concurrent_d_status=$? +wait "$concurrent_e_pid" +concurrent_e_status=$? +set -e +cat "$tmp/concurrent-root-d.jsonl" "$tmp/concurrent-root-e.jsonl" +if [ "$concurrent_d_status" -ne 0 ] || [ "$concurrent_e_status" -ne 0 ]; then + exit 1 +fi +concurrent_end="$(now_ms)" +printf '{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"%s","phase":"concurrent-pair","status":"ok","durationMs":%s,"roots":2}\n' \ + "$platform" "$((concurrent_end - concurrent_start))" +measure_combined concurrent-two-roots "$store" "$tmp/root-d/node_modules" "$tmp/root-e/node_modules" + +test -d "$tmp/root-a/node_modules/.pnpm" +test -d "$tmp/root-b/node_modules/.pnpm" +test -d "$tmp/root-d/node_modules/.pnpm" +test -d "$tmp/root-e/node_modules/.pnpm" +test "$(cd "$tmp/root-a/packages/@overeng/utils" && "$node_bin" -p "require.resolve('effect')")" != \ + "$(cd "$tmp/root-b/packages/@overeng/utils" && "$node_bin" -p "require.resolve('effect')")" + +printf '{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"%s","phase":"correctness","status":"ok","distinctVirtualStores":true,"concurrentRoots":2,"ignoreScriptsConfigured":true,"sigkill137Accepted":false}\n' "$platform" diff --git a/context/dependency-materialization/07-verification/evidence/prune-capability-dev3.jsonl b/context/dependency-materialization/07-verification/evidence/prune-capability-dev3.jsonl new file mode 100644 index 0000000000..bc7a3e10c4 --- /dev/null +++ b/context/dependency-materialization/07-verification/evidence/prune-capability-dev3.jsonl @@ -0,0 +1 @@ +{"schema":"dependency-materialization-verification/v0","kind":"host-capability","surface":"pnpm-store-prune","host":"dev3","platform":"x86_64-linux","filesystem":"ext4","pnpm":"11.8.0","pnpmBin":"/nix/store/m2p5439c2a7dg42hv1fx9igmajdj445x-pnpm-11.8.0/bin/pnpm","packageImportMethod":"auto","rootBase":"/home/schickling/.megarepo","storeBase":"/home/schickling/.local/share/pnpm","rootDevice":65024,"storeDevice":65024,"rootInode":64766128,"rootNlink":2,"storeAliasFound":true,"liveRootSurvivedPrune":true,"removedRootCacheEvicted":true,"beforeKiB":1076,"afterLivePruneKiB":1076,"afterRemovedPruneKiB":1056,"destructivePruneSafe":true} diff --git a/context/dependency-materialization/07-verification/evidence/prune-capability-dev4.jsonl b/context/dependency-materialization/07-verification/evidence/prune-capability-dev4.jsonl new file mode 100644 index 0000000000..44320055e2 --- /dev/null +++ b/context/dependency-materialization/07-verification/evidence/prune-capability-dev4.jsonl @@ -0,0 +1 @@ +{"schema":"dependency-materialization-verification/v0","kind":"host-capability","surface":"pnpm-store-prune","host":"dev4","platform":"aarch64-linux","filesystem":"ext4","pnpm":"11.8.0","pnpmBin":"/nix/store/ng4976sdavaf902xm747xkanwdi4s0cq-pnpm-11.8.0/bin/pnpm","packageImportMethod":"auto","rootBase":"/home/schickling/.megarepo","storeBase":"/home/schickling/.local/share/pnpm","rootDevice":2050,"storeDevice":2050,"rootInode":13911875,"rootNlink":2,"storeAliasFound":true,"liveRootSurvivedPrune":true,"removedRootCacheEvicted":true,"beforeKiB":1076,"afterLivePruneKiB":1076,"afterRemovedPruneKiB":1056,"destructivePruneSafe":true} diff --git a/context/dependency-materialization/07-verification/evidence/storage-sharing-default-v2-darwin.jsonl b/context/dependency-materialization/07-verification/evidence/storage-sharing-default-v2-darwin.jsonl new file mode 100644 index 0000000000..34f85febdc --- /dev/null +++ b/context/dependency-materialization/07-verification/evidence/storage-sharing-default-v2-darwin.jsonl @@ -0,0 +1,24 @@ +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"provenance","status":"ok","implementationHead":"02eb2b77a7beb02936db659383877bfa7040e633","implementationTree":"75cb4074b8fd82f4e4077a797079ab1f9ef7d397","harnessSha256":"c5c0fead91f738aa1fd31462dc6c18a5ff41fc06a370da4d47bd377f30110f88","sourceMode":"verified-git-archive-tree","filesystem":"apfs","pnpm":"11.8.0","node":"v24.15.0"} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"cold-root-a","status":"ok","durationMs":13309,"reused":12,"downloads":600,"teardownExit":134,"completedMaterializationEvidence":true} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:cold-root-a-root-node-modules","status":"ok","sizes":{"physicalKiB":1141552,"apparentKiB":1005795,"files":49833}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:cold-root-a-store","status":"ok","sizes":{"physicalKiB":1025868,"apparentKiB":916366,"files":40239}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"second-root-b","status":"ok","durationMs":3397,"reused":612,"downloads":0,"teardownExit":0,"completedMaterializationEvidence":false} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:second-root-b-root-node-modules","status":"ok","sizes":{"physicalKiB":1141552,"apparentKiB":1005795,"files":49833}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:second-root-b-store","status":"ok","sizes":{"physicalKiB":1016780,"apparentKiB":907350,"files":40237}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:shared-two-roots","status":"ok","sizes":{"physicalKiB":3299884,"apparentKiB":2918939,"files":139903}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"warm-root-b","status":"ok","durationMs":679,"reused":12,"downloads":0,"teardownExit":134,"completedMaterializationEvidence":true} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:warm-root-b-root-node-modules","status":"ok","sizes":{"physicalKiB":1141788,"apparentKiB":1005915,"files":49892}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:warm-root-b-store","status":"ok","sizes":{"physicalKiB":1016812,"apparentKiB":907382,"files":40239}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"isolated-root-c","status":"ok","durationMs":10326,"reused":12,"downloads":600,"teardownExit":134,"completedMaterializationEvidence":true} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:isolated-root-c-root-node-modules","status":"ok","sizes":{"physicalKiB":1141552,"apparentKiB":1005795,"files":49833}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:isolated-root-c-store","status":"ok","sizes":{"physicalKiB":1025516,"apparentKiB":916024,"files":40239}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:isolated-one-root","status":"ok","sizes":{"physicalKiB":2167068,"apparentKiB":1921819,"files":90072}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"concurrent-root-d","status":"ok","durationMs":8603,"reused":612,"downloads":0,"teardownExit":0,"completedMaterializationEvidence":false} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:concurrent-root-d-root-node-modules","status":"ok","sizes":{"physicalKiB":1141552,"apparentKiB":1005795,"files":49833}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:concurrent-root-d-store","status":"ok","sizes":{"physicalKiB":1016780,"apparentKiB":907350,"files":40237}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"concurrent-root-e","status":"ok","durationMs":8420,"reused":612,"downloads":0,"teardownExit":0,"completedMaterializationEvidence":false} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:concurrent-root-e-root-node-modules","status":"ok","sizes":{"physicalKiB":1141552,"apparentKiB":1005795,"files":49833}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:concurrent-root-e-store","status":"ok","sizes":{"physicalKiB":1016780,"apparentKiB":907350,"files":40237}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"concurrent-pair","status":"ok","durationMs":10177,"roots":2} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"size:concurrent-two-roots","status":"ok","sizes":{"physicalKiB":3299884,"apparentKiB":2918939,"files":139903}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"aarch64-darwin","phase":"correctness","status":"ok","distinctVirtualStores":true,"concurrentRoots":2,"ignoreScriptsConfigured":true,"sigkill137Accepted":false} diff --git a/context/dependency-materialization/07-verification/evidence/storage-sharing-default-v2-linux.jsonl b/context/dependency-materialization/07-verification/evidence/storage-sharing-default-v2-linux.jsonl new file mode 100644 index 0000000000..cd85a92e35 --- /dev/null +++ b/context/dependency-materialization/07-verification/evidence/storage-sharing-default-v2-linux.jsonl @@ -0,0 +1,24 @@ +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"provenance","status":"ok","implementationHead":"02eb2b77a7beb02936db659383877bfa7040e633","implementationTree":"75cb4074b8fd82f4e4077a797079ab1f9ef7d397","harnessSha256":"c5c0fead91f738aa1fd31462dc6c18a5ff41fc06a370da4d47bd377f30110f88","sourceMode":"git-worktree","filesystem":"ext4","pnpm":"11.8.0","node":"v24.15.0"} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"cold-root-a","status":"ok","durationMs":10384,"reused":12,"downloads":600,"teardownExit":0,"completedMaterializationEvidence":false} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:cold-root-a-root-node-modules","status":"ok","sizes":{"physicalKiB":1072312,"apparentKiB":927138,"files":49833}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:cold-root-a-store","status":"ok","sizes":{"physicalKiB":1026576,"apparentKiB":907346,"files":40237}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"second-root-b","status":"ok","durationMs":1746,"reused":612,"downloads":0,"teardownExit":0,"completedMaterializationEvidence":false} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:second-root-b-root-node-modules","status":"ok","sizes":{"physicalKiB":1072316,"apparentKiB":927138,"files":49833}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:second-root-b-store","status":"ok","sizes":{"physicalKiB":1026688,"apparentKiB":907346,"files":40237}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:shared-two-roots","status":"ok","sizes":{"physicalKiB":1324272,"apparentKiB":1127826,"files":139903}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"warm-root-b","status":"ok","durationMs":1013,"reused":12,"downloads":0,"teardownExit":0,"completedMaterializationEvidence":false} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:warm-root-b-root-node-modules","status":"ok","sizes":{"physicalKiB":1073004,"apparentKiB":927255,"files":49892}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:warm-root-b-store","status":"ok","sizes":{"physicalKiB":1026832,"apparentKiB":907346,"files":40237}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"isolated-root-c","status":"ok","durationMs":10364,"reused":12,"downloads":600,"teardownExit":0,"completedMaterializationEvidence":false} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:isolated-root-c-root-node-modules","status":"ok","sizes":{"physicalKiB":1072576,"apparentKiB":927138,"files":49833}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:isolated-root-c-store","status":"ok","sizes":{"physicalKiB":1026908,"apparentKiB":907354,"files":40237}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:isolated-one-root","status":"ok","sizes":{"physicalKiB":1175696,"apparentKiB":1017594,"files":90070}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"concurrent-root-d","status":"ok","durationMs":1917,"reused":612,"downloads":0,"teardownExit":0,"completedMaterializationEvidence":false} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:concurrent-root-d-root-node-modules","status":"ok","sizes":{"physicalKiB":1072640,"apparentKiB":927138,"files":49833}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:concurrent-root-d-store","status":"ok","sizes":{"physicalKiB":1026960,"apparentKiB":907346,"files":40237}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"concurrent-root-e","status":"ok","durationMs":1918,"reused":612,"downloads":0,"teardownExit":0,"completedMaterializationEvidence":false} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:concurrent-root-e-root-node-modules","status":"ok","sizes":{"physicalKiB":1072660,"apparentKiB":927138,"files":49833}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:concurrent-root-e-store","status":"ok","sizes":{"physicalKiB":1026960,"apparentKiB":907346,"files":40237}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"concurrent-pair","status":"ok","durationMs":2512,"roots":2} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"size:concurrent-two-roots","status":"ok","sizes":{"physicalKiB":1324556,"apparentKiB":1127826,"files":139903}} +{"schema":"dependency-materialization-verification/v0","kind":"benchmark","surface":"storage-sharing","platform":"x86_64-linux","phase":"correctness","status":"ok","distinctVirtualStores":true,"concurrentRoots":2,"ignoreScriptsConfigured":true,"sigkill137Accepted":false} diff --git a/context/dependency-materialization/07-verification/evidence/storage-sharing-default-v2.json b/context/dependency-materialization/07-verification/evidence/storage-sharing-default-v2.json new file mode 100644 index 0000000000..792f4584ce --- /dev/null +++ b/context/dependency-materialization/07-verification/evidence/storage-sharing-default-v2.json @@ -0,0 +1,262 @@ +{ + "schema": "dependency-materialization-verification/v0", + "kind": "default-gate", + "surface": "storage-sharing", + "policy": { + "packageStoreScope": "host-user", + "virtualStoreScope": "materialization-root", + "packageImportMethod": "auto", + "enableGlobalVirtualStore": false + }, + "measurementMethod": { + "date": "2026-07-18", + "workspace": "effect-utils", + "implementationHead": "02eb2b77a7beb02936db659383877bfa7040e633", + "implementationTree": "75cb4074b8fd82f4e4077a797079ab1f9ef7d397", + "harness": { + "path": "context/dependency-materialization/07-verification/evidence/measure-storage-sharing-default.sh", + "sha256": "c5c0fead91f738aa1fd31462dc6c18a5ff41fc06a370da4d47bd377f30110f88" + }, + "rawArtifacts": { + "linux": { + "path": "storage-sharing-default-v2-linux.jsonl", + "sha256": "416c08cd585da28707ccc5f4ef41eb3efbf8367895396c4919cb9b266587bae5" + }, + "darwin": { + "path": "storage-sharing-default-v2-darwin.jsonl", + "sha256": "9882c71f64cf916a5ca8f29a4f63f4c86361a089840f257e42a3add1d5bbec92" + } + }, + "sourcePreparation": { + "linux": "five detached Git worktrees at implementationHead", + "darwin": "git archive whose reconstructed Git tree is verified against implementationTree on mbp2025" + }, + "packageManager": "pnpm@11.8.0", + "installPolicy": { + "frozenLockfile": true, + "ignoreScripts": true, + "sideEffectsCache": false, + "verifyStoreIntegrity": true, + "strictStorePackageContentCheck": true, + "childConcurrency": 1, + "networkConcurrency": 4 + }, + "phases": [ + "empty shared store into root A", + "same shared store into root B", + "warm root B", + "empty isolated store into root C", + "simultaneous roots D and E against the populated shared store" + ], + "sizeMethod": "du allocated-block and apparent KiB plus regular-file counts; physicalKiB is an allocated-block comparator, not APFS unique-extent accounting; the two-isolated-root comparator is twice the measured one-root isolated baseline" + }, + "hostLifecycleCapability": { + "harness": { + "path": "context/dependency-materialization/07-verification/evidence/measure-prune-capability.sh", + "sha256": "646491a8f29146b1f60524c646c15cbc8d80e03c62f89b403cd15bb975ced6b3" + }, + "hosts": { + "dev3": { + "rawArtifact": "prune-capability-dev3.jsonl", + "sha256": "9ddcc46802efaf137e4d4207686e057e5e6123a7a0917d68d75b284495e713c1", + "platform": "x86_64-linux", + "filesystem": "ext4", + "pnpm": "11.8.0", + "rootBase": "/home/schickling/.megarepo", + "storeBase": "/home/schickling/.local/share/pnpm", + "rootDevice": 65024, + "storeDevice": 65024, + "rootNlink": 2, + "liveRootSurvivedPrune": true, + "removedRootCacheEvicted": true, + "destructivePruneSafe": true + }, + "dev4": { + "rawArtifact": "prune-capability-dev4.jsonl", + "sha256": "2d80e956364516f19b65c8752e81619d368dc13578ffd196fea8317eed1c666f", + "platform": "aarch64-linux", + "filesystem": "ext4", + "pnpm": "11.8.0", + "rootBase": "/home/schickling/.megarepo", + "storeBase": "/home/schickling/.local/share/pnpm", + "rootDevice": 2050, + "storeDevice": 2050, + "rootNlink": 2, + "liveRootSurvivedPrune": true, + "removedRootCacheEvicted": true, + "destructivePruneSafe": true + } + } + }, + "platforms": [ + { + "platform": "x86_64-linux", + "status": "ok", + "environment": { + "filesystem": "ext4", + "node": "v24.15.0" + }, + "phaseMatrix": { + "coldSharedRootA": { + "durationMs": 10384, + "reused": 12, + "downloaded": 600, + "rootNodeModules": { + "physicalKiB": 1072312, + "apparentKiB": 927138, + "files": 49833 + }, + "store": { + "physicalKiB": 1026576, + "apparentKiB": 907346, + "files": 40237 + } + }, + "secondSharedRootB": { + "durationMs": 1746, + "reused": 612, + "downloaded": 0 + }, + "warmSharedRootB": { + "durationMs": 1013, + "reused": 12, + "downloaded": 0 + }, + "isolatedRootC": { + "durationMs": 10364, + "reused": 12, + "downloaded": 600, + "combinedStoreAndRoot": { + "physicalKiB": 1175696, + "apparentKiB": 1017594, + "files": 90070 + } + }, + "concurrentRootsDAndE": { + "durationMs": 2512, + "rootDDurationMs": 1917, + "rootEDurationMs": 1918, + "eachReused": 612, + "eachDownloaded": 0, + "combinedStoreAndRoots": { + "physicalKiB": 1324556, + "apparentKiB": 1127826, + "files": 139903 + } + } + }, + "twoRootComparison": { + "sharedMeasured": { + "physicalKiB": 1324272, + "apparentKiB": 1127826, + "files": 139903 + }, + "isolatedComparator": { + "physicalKiB": 2351392, + "apparentKiB": 2035188, + "files": 180140 + }, + "improvementPercent": { + "physicalKiB": 43.7, + "apparentKiB": 44.6, + "files": 22.3, + "secondRootDuration": 83.2 + } + }, + "correctness": { + "distinctVirtualStores": true, + "concurrentRoots": 2, + "ignoreScriptsConfigured": true + } + }, + { + "platform": "aarch64-darwin", + "status": "ok", + "environment": { + "host": "mbp2025", + "filesystem": "apfs", + "node": "v24.15.0" + }, + "phaseMatrix": { + "coldSharedRootA": { + "durationMs": 13309, + "reused": 12, + "downloaded": 600, + "teardownExit": 134, + "completedMaterializationEvidence": true, + "rootNodeModules": { + "physicalKiB": 1141552, + "apparentKiB": 1005795, + "files": 49833 + }, + "store": { + "physicalKiB": 1025868, + "apparentKiB": 916366, + "files": 40239 + } + }, + "secondSharedRootB": { + "durationMs": 3397, + "reused": 612, + "downloaded": 0, + "teardownExit": 0 + }, + "warmSharedRootB": { + "durationMs": 679, + "reused": 12, + "downloaded": 0, + "teardownExit": 134, + "completedMaterializationEvidence": true + }, + "isolatedRootC": { + "durationMs": 10326, + "reused": 12, + "downloaded": 600, + "teardownExit": 134, + "completedMaterializationEvidence": true, + "combinedStoreAndRoot": { + "physicalKiB": 2167068, + "apparentKiB": 1921819, + "files": 90072 + } + }, + "concurrentRootsDAndE": { + "durationMs": 10177, + "rootDDurationMs": 8603, + "rootEDurationMs": 8420, + "eachReused": 612, + "eachDownloaded": 0, + "combinedStoreAndRoots": { + "physicalKiB": 3299884, + "apparentKiB": 2918939, + "files": 139903 + } + } + }, + "twoRootComparison": { + "sharedMeasured": { + "physicalKiB": 3299884, + "apparentKiB": 2918939, + "files": 139903 + }, + "isolatedComparator": { + "physicalKiB": 4334136, + "apparentKiB": 3843638, + "files": 180144 + }, + "improvementPercent": { + "physicalKiB": 23.9, + "apparentKiB": 24.1, + "files": 22.3, + "secondRootDuration": 67.1 + } + }, + "correctness": { + "distinctVirtualStores": true, + "concurrentRoots": 2, + "ignoreScriptsConfigured": true, + "sigkill137Accepted": false + } + } + ] +} diff --git a/context/dependency-materialization/07-verification/evidence/validate-storage-sharing-default.mjs b/context/dependency-materialization/07-verification/evidence/validate-storage-sharing-default.mjs new file mode 100644 index 0000000000..59974416bf --- /dev/null +++ b/context/dependency-materialization/07-verification/evidence/validate-storage-sharing-default.mjs @@ -0,0 +1,302 @@ +import { createHash } from 'node:crypto' +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const directory = dirname(fileURLToPath(import.meta.url)) +const read = (path) => readFileSync(join(directory, path), 'utf8') +const sha256 = (path) => createHash('sha256').update(read(path)).digest('hex') +const parseJsonl = (path) => + read(path) + .trim() + .split('\n') + .map((line) => JSON.parse(line)) +// Assertion helpers form a compact verification DSL; positional actual/expected/label +// call sites are more legible here than repeated object literals. +// oxlint-disable-next-line overeng/named-args +const assertEqual = (actual, expected, label) => { + if (actual !== expected) throw new Error(`${label}: expected ${expected}, got ${actual}`) +} +// oxlint-disable-next-line overeng/named-args +const assertTrue = (value, label) => assertEqual(value, true, label) +const round1 = (value) => Math.round(value * 10) / 10 +const sizeKeys = ['physicalKiB', 'apparentKiB', 'files'] +const installPhases = [ + 'cold-root-a', + 'second-root-b', + 'warm-root-b', + 'isolated-root-c', + 'concurrent-root-d', + 'concurrent-root-e', +] +const sizePhases = [ + 'cold-root-a-root-node-modules', + 'cold-root-a-store', + 'second-root-b-root-node-modules', + 'second-root-b-store', + 'shared-two-roots', + 'warm-root-b-root-node-modules', + 'warm-root-b-store', + 'isolated-root-c-root-node-modules', + 'isolated-root-c-store', + 'isolated-one-root', + 'concurrent-root-d-root-node-modules', + 'concurrent-root-d-store', + 'concurrent-root-e-root-node-modules', + 'concurrent-root-e-store', + 'concurrent-two-roots', +].map((name) => `size:${name}`) +const requiredPhases = [ + 'provenance', + ...installPhases, + ...sizePhases, + 'concurrent-pair', + 'correctness', +] + +const summary = JSON.parse(read('storage-sharing-default-v2.json')) +assertEqual(summary.schema, 'dependency-materialization-verification/v0', 'summary schema') +assertEqual(summary.kind, 'default-gate', 'summary kind') +assertEqual(summary.surface, 'storage-sharing', 'summary surface') +assertEqual(summary.policy.packageStoreScope, 'host-user', 'package store scope') +assertEqual(summary.policy.virtualStoreScope, 'materialization-root', 'virtual store scope') +assertEqual(summary.policy.packageImportMethod, 'auto', 'import method') +assertEqual(summary.policy.enableGlobalVirtualStore, false, 'GVS policy') +assertEqual( + sha256('measure-storage-sharing-default.sh'), + summary.measurementMethod.harness.sha256, + 'storage harness digest', +) +assertEqual( + sha256('measure-prune-capability.sh'), + summary.hostLifecycleCapability.harness.sha256, + 'prune harness digest', +) +const expectedPnpm = summary.measurementMethod.packageManager.replace(/^pnpm@/, '') +const platformInputs = { + 'x86_64-linux': { + artifact: summary.measurementMethod.rawArtifacts.linux, + summary: summary.platforms.find((item) => item.platform === 'x86_64-linux'), + filesystem: 'ext4', + sourceMode: 'git-worktree', + }, + 'aarch64-darwin': { + artifact: summary.measurementMethod.rawArtifacts.darwin, + summary: summary.platforms.find((item) => item.platform === 'aarch64-darwin'), + filesystem: 'apfs', + sourceMode: 'verified-git-archive-tree', + }, +} +assertEqual(summary.platforms.length, Object.keys(platformInputs).length, 'platform count') + +for (const [platform, input] of Object.entries(platformInputs)) { + assertTrue(input.summary !== undefined, `${platform} summary exists`) + assertEqual(input.summary.status, 'ok', `${platform} summary status`) + assertEqual( + input.summary.environment.filesystem, + input.filesystem, + `${platform} summary filesystem`, + ) + assertEqual(sha256(input.artifact.path), input.artifact.sha256, `${platform} raw digest`) + const records = parseJsonl(input.artifact.path) + const byPhase = new Map(records.map((record) => [record.phase, record])) + assertEqual(byPhase.size, records.length, `${platform} unique phases`) + assertEqual(records.length, requiredPhases.length, `${platform} record count`) + for (const phaseName of requiredPhases) + assertTrue(byPhase.has(phaseName), `${platform} ${phaseName}`) + for (const record of records) { + assertEqual(record.schema, summary.schema, `${platform} ${record.phase} schema`) + assertEqual(record.kind, 'benchmark', `${platform} ${record.phase} kind`) + assertEqual(record.surface, summary.surface, `${platform} ${record.phase} surface`) + assertEqual(record.platform, platform, `${platform} ${record.phase} platform`) + assertEqual(record.status, 'ok', `${platform} ${record.phase} status`) + } + + const provenance = byPhase.get('provenance') + assertEqual( + provenance.implementationHead, + summary.measurementMethod.implementationHead, + `${platform} implementation head`, + ) + assertEqual( + provenance.implementationTree, + summary.measurementMethod.implementationTree, + `${platform} implementation tree`, + ) + assertEqual( + provenance.harnessSha256, + summary.measurementMethod.harness.sha256, + `${platform} harness provenance`, + ) + assertEqual(provenance.sourceMode, input.sourceMode, `${platform} source mode`) + assertEqual(provenance.filesystem, input.filesystem, `${platform} raw filesystem`) + assertEqual(provenance.pnpm, expectedPnpm, `${platform} pnpm`) + assertEqual(provenance.node, input.summary.environment.node, `${platform} node`) + + const phase = input.summary.phaseMatrix + const mappings = [ + ['coldSharedRootA', 'cold-root-a'], + ['secondSharedRootB', 'second-root-b'], + ['warmSharedRootB', 'warm-root-b'], + ['isolatedRootC', 'isolated-root-c'], + ] + for (const [summaryName, rawName] of mappings) { + const raw = byPhase.get(rawName) + const value = phase[summaryName] + assertEqual(raw.durationMs, value.durationMs, `${platform} ${rawName} duration`) + assertEqual(raw.reused, value.reused, `${platform} ${rawName} reused`) + assertEqual(raw.downloads, value.downloaded, `${platform} ${rawName} downloads`) + const acceptedAbort = raw.teardownExit === 134 && raw.completedMaterializationEvidence === true + assertTrue( + (raw.teardownExit === 0 && raw.completedMaterializationEvidence === false) || acceptedAbort, + `${platform} ${rawName} teardown semantics`, + ) + if (platform === 'aarch64-darwin') { + assertEqual(raw.teardownExit, value.teardownExit, `${platform} ${rawName} teardown exit`) + assertEqual( + raw.completedMaterializationEvidence, + value.completedMaterializationEvidence ?? false, + `${platform} ${rawName} completion evidence`, + ) + } else { + assertEqual(raw.teardownExit, 0, `${platform} ${rawName} teardown exit`) + } + } + for (const rawName of ['concurrent-root-d', 'concurrent-root-e']) { + const raw = byPhase.get(rawName) + assertEqual(raw.teardownExit, 0, `${platform} ${rawName} teardown exit`) + assertEqual( + raw.completedMaterializationEvidence, + false, + `${platform} ${rawName} completion evidence`, + ) + } + + const concurrent = phase.concurrentRootsDAndE + const concurrentPair = byPhase.get('concurrent-pair') + assertEqual(concurrentPair.roots, 2, `${platform} concurrent pair roots`) + assertEqual(concurrentPair.durationMs, concurrent.durationMs, `${platform} concurrent duration`) + assertEqual( + byPhase.get('concurrent-root-d').durationMs, + concurrent.rootDDurationMs, + `${platform} concurrent root D duration`, + ) + assertEqual( + byPhase.get('concurrent-root-e').durationMs, + concurrent.rootEDurationMs, + `${platform} concurrent root E duration`, + ) + for (const name of ['concurrent-root-d', 'concurrent-root-e']) { + assertEqual(byPhase.get(name).reused, concurrent.eachReused, `${platform} ${name} reused`) + assertEqual( + byPhase.get(name).downloads, + concurrent.eachDownloaded, + `${platform} ${name} downloads`, + ) + } + + for (const name of sizePhases) { + for (const key of sizeKeys) { + assertTrue( + Number.isSafeInteger(byPhase.get(name).sizes[key]), + `${platform} ${name} ${key} integer`, + ) + assertTrue(byPhase.get(name).sizes[key] > 0, `${platform} ${name} ${key} positive`) + } + } + const sizeMappings = [ + ['size:cold-root-a-root-node-modules', phase.coldSharedRootA.rootNodeModules], + ['size:cold-root-a-store', phase.coldSharedRootA.store], + ['size:isolated-one-root', phase.isolatedRootC.combinedStoreAndRoot], + ['size:shared-two-roots', input.summary.twoRootComparison.sharedMeasured], + ['size:concurrent-two-roots', concurrent.combinedStoreAndRoots], + ] + for (const [rawName, value] of sizeMappings) { + for (const key of sizeKeys) { + assertEqual(byPhase.get(rawName).sizes[key], value[key], `${platform} ${rawName} ${key}`) + } + } + const isolated = byPhase.get('size:isolated-one-root').sizes + const shared = byPhase.get('size:shared-two-roots').sizes + for (const key of sizeKeys) { + assertEqual( + isolated[key] * 2, + input.summary.twoRootComparison.isolatedComparator[key], + `${platform} isolated comparator ${key}`, + ) + assertEqual( + round1(((isolated[key] * 2 - shared[key]) / (isolated[key] * 2)) * 100), + input.summary.twoRootComparison.improvementPercent[key], + `${platform} improvement ${key}`, + ) + } + assertEqual( + round1( + ((phase.isolatedRootC.durationMs - phase.secondSharedRootB.durationMs) / + phase.isolatedRootC.durationMs) * + 100, + ), + input.summary.twoRootComparison.improvementPercent.secondRootDuration, + `${platform} second-root improvement`, + ) + + const correctness = byPhase.get('correctness') + for (const key of [ + 'distinctVirtualStores', + 'concurrentRoots', + 'ignoreScriptsConfigured', + 'sigkill137Accepted', + ]) { + assertEqual( + correctness[key], + input.summary.correctness[key] ?? false, + `${platform} correctness ${key}`, + ) + } + assertEqual(correctness.distinctVirtualStores, true, `${platform} distinct virtual stores`) + assertEqual(correctness.concurrentRoots, 2, `${platform} concurrent roots`) + assertEqual(correctness.ignoreScriptsConfigured, true, `${platform} ignore scripts configuration`) + assertEqual(correctness.sigkill137Accepted, false, `${platform} SIGKILL policy`) +} + +for (const [host, capability] of Object.entries(summary.hostLifecycleCapability.hosts)) { + assertEqual(sha256(capability.rawArtifact), capability.sha256, `${host} raw digest`) + const records = parseJsonl(capability.rawArtifact) + assertEqual(records.length, 1, `${host} record count`) + const [raw] = records + assertEqual(raw.schema, summary.schema, `${host} schema`) + assertEqual(raw.kind, 'host-capability', `${host} kind`) + assertEqual(raw.surface, 'pnpm-store-prune', `${host} surface`) + for (const key of [ + 'host', + 'platform', + 'filesystem', + 'pnpm', + 'rootBase', + 'storeBase', + 'rootDevice', + 'storeDevice', + 'rootNlink', + 'liveRootSurvivedPrune', + 'removedRootCacheEvicted', + 'destructivePruneSafe', + ]) { + assertEqual(raw[key], capability[key] ?? host, `${host} ${key}`) + } + assertEqual(raw.pnpm, expectedPnpm, `${host} deployed pnpm version`) + assertTrue(raw.pnpmBin.endsWith(`-pnpm-${expectedPnpm}/bin/pnpm`), `${host} pnpm derivation`) + assertEqual(raw.packageImportMethod, summary.policy.packageImportMethod, `${host} import method`) + assertEqual(raw.rootDevice, raw.storeDevice, `${host} same device`) + assertTrue(raw.rootNlink >= 2, `${host} hardlink count`) + assertEqual(raw.storeAliasFound, true, `${host} store inode alias`) + assertEqual(raw.beforeKiB, raw.afterLivePruneKiB, `${host} live-root prune preserves cache`) + assertTrue( + raw.afterRemovedPruneKiB < raw.afterLivePruneKiB, + `${host} removed-root prune reclaims cache`, + ) + assertEqual(raw.liveRootSurvivedPrune, true, `${host} live root survives`) + assertEqual(raw.removedRootCacheEvicted, true, `${host} removed root evicted`) + assertEqual(raw.destructivePruneSafe, true, `${host} destructive prune safety`) +} + +console.log('storage-sharing-default-v2: ok') diff --git a/context/dependency-materialization/07-verification/requirements.md b/context/dependency-materialization/07-verification/requirements.md index e2ac467d3b..ef6ebf9468 100644 --- a/context/dependency-materialization/07-verification/requirements.md +++ b/context/dependency-materialization/07-verification/requirements.md @@ -3,9 +3,9 @@ ## Context Verification defines the proof, benchmark, and regression architecture for -dependency materialization. It refines DMP-R16 through DMP-R20 and composes the -live pnpm, projection, Nix prepared-deps, store-authority, Buck2 evidence, and -observability subsystems. +dependency materialization. It refines DMP-R11 and DMP-R16 through DMP-R20 and +composes the live pnpm, projection, Nix prepared-deps, store-authority, Buck2 +evidence, and observability subsystems. ## Assumptions @@ -29,8 +29,8 @@ observability subsystems. ### Must cover correctness - **DMP.VER-R01 Fixture regressions:** Unit and smoke fixtures must cover - profile identity, strict install rejection, projection health, native package - classification, and doctor/repair decisions. + Materialization Profile identity, strict install rejection, projection + health, native package classification, and doctor/repair decisions. Refines: DMP-R16, DMP-R17, DMP-R20. - **DMP.VER-R02 Negative lifecycle proof:** At least one fixture must prove that managed materialization does not run dependency lifecycle scripts, @@ -40,16 +40,19 @@ observability subsystems. must have fixtures that reject `.bin`, unexpected native output, known platform package directories, and leaked package-manager state. Refines: DMP-R05, DMP-R08, DMP-R18. -- **DMP.VER-R04 Shared-store failure proof:** Store-authority changes must - preserve a proof that raw profile-local prune can break sibling offline - reinstall for shared pools and that coordinated repair targets every root. - Refines: DMP-R13, DMP-R14, DMP-R15. +- **DMP.VER-R04 Store Cache eviction proof:** Store-authority changes must + preserve a proof that Store Cache eviction can break a future offline + reinstall while leaving an already-materialized root healthy, and prove that + effect-utils-managed root repair does not prune the host cache. The proof must + not imply that root health establishes offline readiness. + Refines: DMP-R13, DMP-R14, DMP.STORE-R05. ### Must cover performance and sharing -- **DMP.VER-R05 Benchmark matrix:** Store-trait changes must record cold, - warm, offline, concurrent, byte, file-count, and repair metrics. - Refines: DMP-R16, DMP-R19, DMP.STORE-R08. +- **DMP.VER-R05 Benchmark matrix:** Storage-sharing changes must record cold, + warm, concurrent, byte, and file-count metrics. Offline and repair metrics + are required only for an explicit offline-readiness or repair claim. + Refines: DMP-R16, DMP-R19, DMP.STORE-R13. - **DMP.VER-R06 Real-workload gate:** Default changes require at least one downstream real graph for each affected platform class, or an explicit pending-system marker that prevents overgeneralized conclusions. @@ -57,13 +60,13 @@ observability subsystems. - **DMP.VER-R07 Cache-efficiency comparison:** Claims about host-wide sharing must compare against an isolated baseline on the same graph and machine class. - Refines: DMP-R16, DMP.STORE-R09. + Refines: DMP-R16, DMP.STORE-R14. ### Must be auditable - **DMP.VER-R08 Machine-readable evidence:** Proofs and benchmarks must emit - stable records for status, inputs, platform, store trait, timings, sizes, and - skip reasons. + stable records for status, inputs, platform, writable-state scope, + content-pool scope, timings, sizes, and skip reasons. Refines: DMP-R19, DMP.OBS-R01, DMP.OBS-R02. - **DMP.VER-R09 Decision linkage:** Consequential DMP decisions must name the evidence category that justifies them and any evidence still pending. @@ -73,3 +76,24 @@ observability subsystems. requirement, implemented as a reusable proof or benchmark, recorded as pending evidence, or explicitly rejected with rationale. Refines: DMP-R20. + +### Must prove dependency identity + +- **DMP.VER-R11 Package Instance identity proof:** Changes capable of selecting + or writing Dependency Edge identities must prove that same-name multi-version + and distinct peer-context Package Instances preserve their materializer- + selected identities. A validator that claims topology containment must prove + that it detects a negative out-of-band edge override across Materialization + Roots; it need not reimplement the materializer's exact locator selection. + Install-order permutations are required when mutable dependency state is + reused across installs. + Refines: DMP-R11, DMP.LIVE-R07, DMP.PROJ-R09, DMP-R20. +- **DMP.VER-R12 Topology-reuse comparison:** Repeated topology work must be + measured on the same real workloads and platform classes across root-local, + shared Global Virtual Store, identity-partitioned Global Virtual Store, and + isolated baselines. The comparison must include physical + bytes, repeated work, cold/second/warm/offline/repair/concurrent latency, + graph identity, lock contention, fault injection, and one-root repair scope. + Results may motivate a Hermetic Dependency Artifact but must not override the + purity and authority gates or make a mutable shared topology admissible. + Refines: DMP-R16, DMP-R20, DMP-R22, DMP-R23, DMP-R24, DMP.STORE-R14. diff --git a/context/dependency-materialization/07-verification/spec.md b/context/dependency-materialization/07-verification/spec.md index 5783a13743..9d4b4acb5e 100644 --- a/context/dependency-materialization/07-verification/spec.md +++ b/context/dependency-materialization/07-verification/spec.md @@ -7,12 +7,14 @@ Status: **Draft** ## Requirement Trace -| Section | Requirements | -| ---------------- | -------------------------------------------------- | -| Evidence Tiers | DMP.VER-R01, DMP.VER-R02, DMP.VER-R03, DMP.VER-R04 | -| Benchmark Matrix | DMP.VER-R05, DMP.VER-R06, DMP.VER-R07 | -| Evidence Records | DMP.VER-R08, DMP.VER-R09 | -| Evidence Intake | DMP.VER-R10 | +| Section | Requirements | +| ------------------- | -------------------------------------------------- | +| Evidence Tiers | DMP.VER-R01, DMP.VER-R02, DMP.VER-R03, DMP.VER-R04 | +| Benchmark Matrix | DMP.VER-R05, DMP.VER-R06, DMP.VER-R07 | +| Evidence Records | DMP.VER-R08, DMP.VER-R09 | +| Evidence Intake | DMP.VER-R10 | +| Dependency Identity | DMP.VER-R11 | +| Topology Reuse | DMP.VER-R12 | ## Evidence Tiers @@ -32,15 +34,16 @@ fixture checks ## Correctness Matrix -| Surface | Required evidence | Owning subsystem | -| ---------------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------- | -| Strict pnpm policy | Reject lifecycle/build override flags before pnpm runs; prove sentinel scripts do not run. | [01-live-pnpm](../01-live-pnpm/spec.md) | -| Bin projection | Manifest fixture plus pnpm-linker oracle cases; prove missing/stale bins are repaired without scripts. | [02-projections](../02-projections/spec.md) | -| Prepared deps | Scan fixtures for `.bin`, leaked state, unexpected `*.node`, and known platform dirs. | [03-nix-prepared-deps](../03-nix-prepared-deps/spec.md) | -| Native packages | Lockfile-policy audit and graft-file existence checks. | [03-nix-prepared-deps/02-native-node-packages](../03-nix-prepared-deps/02-native-node-packages/spec.md) | -| Shared store authority | Raw-prune failure repro, doctor refusal, all-root repair plan. | [04-store-authority](../04-store-authority/spec.md) | -| Buck2 evidence | Stable declared-input evidence; no live pnpm mutation. | [05-buck2-evidence](../05-buck2-evidence/spec.md) | -| Observability | Fixture records for phase, timing, size, reuse, profile link, and safe paths. | [06-observability](../06-observability/spec.md) | +| Surface | Required evidence | Owning subsystem | +| ---------------------- | -------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| Strict pnpm policy | Reject lifecycle/build override flags before pnpm runs; prove sentinel scripts do not run. | [01-live-pnpm](../01-live-pnpm/spec.md) | +| Dependency identity | Install incompatible peer graphs in both orders; prove pnpm-selected edges and type identity are stable. | [01-live-pnpm](../01-live-pnpm/spec.md) | +| Bin projection | Manifest fixture plus pnpm-linker oracle cases; prove missing/stale bins are repaired without scripts. | [02-projections](../02-projections/spec.md) | +| Prepared deps | Scan fixtures for `.bin`, leaked state, unexpected `*.node`, and known platform dirs. | [03-nix-prepared-deps](../03-nix-prepared-deps/spec.md) | +| Native packages | Lockfile-policy audit and graft-file existence checks. | [03-nix-prepared-deps/02-native-node-packages](../03-nix-prepared-deps/02-native-node-packages/spec.md) | +| Shared store authority | Root-local topology proof, shared-content immutability, and raw-prune refusal. | [04-store-authority](../04-store-authority/spec.md) | +| Buck2 evidence | Stable declared-input evidence; no live pnpm mutation. | [05-buck2-evidence](../05-buck2-evidence/spec.md) | +| Observability | Fixture records for phase, timing, size, reuse, profile link, and safe paths. | [06-observability](../06-observability/spec.md) | ## Benchmark Matrix @@ -51,10 +54,11 @@ runs can share one parser: { "schema": "dependency-materialization-verification/v0", "kind": "benchmark", - "surface": "store-trait", + "surface": "storage-sharing", "workspace": "effect-utils", "platform": "aarch64-darwin", - "storeTrait": "darwinSplitCas", + "stateScope": "materialization-root", + "contentPoolScope": "host-shared", "phase": "offline-reinstall", "status": "ok", "timingsMs": { "coldA": 1234, "coldB": 640, "offline": 410 }, @@ -70,7 +74,7 @@ why work did not run: { "schema": "dependency-materialization-verification/v0", "kind": "benchmark", - "surface": "store-trait", + "surface": "storage-sharing", "status": "skipped", "reason": "low-disk", "availableGiB": 31, @@ -82,6 +86,35 @@ Default changes require same-workload comparisons against the current default and an isolated baseline. Cache-efficiency claims must report bytes and file counts, not only timing. +To quantify repeated topology work and evaluate a future Hermetic Dependency +Artifact, the same commit, graph, and machine class must compare: + +1. shared Store Cache with root-local virtual stores; +2. shared Store Cache with one shared Global Virtual Store; +3. shared Store Cache with Global Virtual Stores partitioned by declared graph + or lock identity; +4. isolated Store Cache with root-local virtual stores. + +The matrix runs on effect-utils and the real dotfiles/Vista graph across ext4 +and APFS. It records extent-aware physical allocation where available, apparent +bytes, files/inodes, downloads, cold/second/warm/offline/repair/concurrent +latency, lock wait, peer/Package Instance identity, injected missing/corrupt +edges, and the scope required to repair one root. Options that fail purity, +identity, data-safety, concurrency, or bounded-repair gates are inadmissible +regardless of their byte or latency result. + +The measured package-store policy evidence is recorded in +[`evidence/storage-sharing-default-v2.json`](./evidence/storage-sharing-default-v2.json). +It is pinned to its recorded implementation head and harness checksum rather +than silently acting as a current-head performance baseline. It is +machine-validated against committed raw JSONL records and records +separate real-workload Linux/ext4 and Darwin/APFS cold, warm, isolated, +second-root, and concurrent phase matrices. The same evidence bundle carries +per-host hardlink/prune capability records for hosts that enable destructive +maintenance. Results remain platform- and host-specific: neither platform's +timing, allocation, effective import behavior, nor prune safety may be +generalized to another filesystem or host. + ## Evidence Intake And Graduation Verification may import evidence from prototype branches, downstream @@ -100,5 +133,5 @@ outcomes: obsolete assumption. Research may be retired once every durable finding has one of those outcomes. -Historical source links belong in `.research/` or `.experiments/`, not in the -normative spec. +Historical source material belongs in `.reference/`; project-generated +validation evidence belongs in focused `.experiments/` records. diff --git a/context/dependency-materialization/experiments.md b/context/dependency-materialization/experiments.md index d3cf4a04e3..3b6d7db04e 100644 --- a/context/dependency-materialization/experiments.md +++ b/context/dependency-materialization/experiments.md @@ -23,8 +23,9 @@ Result summary: - shared-pool GC must mark from every active root or refuse to sweep; - isolated stores are simpler but lose a large host-wide byte and file-count win; -- Darwin pnpm can exit 134/137 after materialization, so wrappers classify - those exits only after projection health checks. +- Historical Darwin runs produced exits 134/137 after apparent materialization. + The complete shared-store policy normalizes only the recovered exit-134 path; + SIGKILL 137 remains failure until exact shared-index recovery is proven. Conclusion: diff --git a/context/dependency-materialization/glossary.md b/context/dependency-materialization/glossary.md deleted file mode 100644 index 7694dd551c..0000000000 --- a/context/dependency-materialization/glossary.md +++ /dev/null @@ -1,36 +0,0 @@ -# Dependency Materialization Glossary - -**Dependency materialization:** The process of realizing declared dependency -inputs into dependency data, projection state, native integrations, and -evidence that tools can execute against. - -**Dependency data:** Package files linked or restored without running package -lifecycle code. Prepared dependency FODs contain dependency data, not live -package-manager state. - -**Projection state:** Deterministic files derived after dependency data exists, -such as `node_modules/.bin` entries and profile-owned workspace links. - -**Dependency materialization profile:** Versioned evidence that names the -topology, package-manager policy, toolchain inputs, store trait, authorities, -and semantic inputs for one materialization root. - -**Store trait:** The declared storage and sharing strategy for a profile, such -as `ciJobLocal`, `darwinSplitCas`, `linuxSharedHardlink`, `isolated`, -`nixPreparedDeps`, or `frozenSeed`. - -**Shared content pool:** A package-content store used by multiple profiles. -Pruning or garbage-collecting it requires root-set authority. - -**Prepared dependency artifact:** A Nix fixed-output dependency data tree -created from declared inputs and restored into downstream builds. - -**Native graft:** A platform-specific native package output supplied by Nix or -an explicit wrapper after prepared dependency data is restored. - -**Pure package artifact:** Package contents accepted as dependency data without -running lifecycle scripts, downloads, source compilation, or generated native -build output. - -**Buck2 evidence:** A declared dependency-profile fact consumed by Buck2 without -granting Buck2 authority over live pnpm install or shared-store repair. diff --git a/context/dependency-materialization/intuition.md b/context/dependency-materialization/intuition.md index 051c9a76cc..1b68a69b14 100644 --- a/context/dependency-materialization/intuition.md +++ b/context/dependency-materialization/intuition.md @@ -9,18 +9,17 @@ graph is realized through several mechanisms: a live `node_modules` tree during development, a prepared dependency artifact in Nix, a job-local CI install, and eventually Buck2 evidence or actions. -The VRS is shaped around one rule: dependency identity is shared, but mutation -authority is not. +The VRS is shaped around one rule: immutable dependency work may have a shared +identity, but mutable realization state always belongs to one root. ```text - dependency materialization profile - | - +-----------------------+-----------------------+ - | | | - live pnpm state Nix prepared data Buck2 evidence - mutable/repairable immutable/restored declared graph fact - | | | - +----------- projection + observability --------+ + declared inputs prepared profileKey + | | + v v + live pnpm root Nix data + Buck2 evidence + mutable/repairable immutable/declared + | + +---- root-owned projection + observability ``` pnpm may resolve and link package data, but effect-utils-managed paths do not diff --git a/context/dependency-materialization/ontology.md b/context/dependency-materialization/ontology.md new file mode 100644 index 0000000000..72e639fa98 --- /dev/null +++ b/context/dependency-materialization/ontology.md @@ -0,0 +1,180 @@ +# Dependency Materialization Ontology + +This ontology owns the vocabulary shared by the dependency-materialization +subsystems. Subsystem documents inherit these terms and add only +realization-specific language. + +## Federation + +### Inherited language + +- From package managers: **package**, **dependency**, **peer dependency**, + **manifest**, **lockfile**, and **workspace**. +- From pnpm: **package snapshot** and **virtual store**. A pnpm package snapshot + is a pnpm representation; it is not the cross-realization identity defined + below. +- From Nix: **derivation**, **fixed-output derivation**, and **Nix store**. +- From graph theory: **graph**, **node**, and **edge**. + +### Owned language + +This context owns **Dependency Materialization**, **Materialization Root**, +**Materialization Profile**, **Authoritative Materializer**, **Dependency +Graph**, **Package Instance**, **Dependency Edge**, **Dependency Data**, +**Projection State**, **Store Cache**, and **Repair**. + +### Subsystem language + +Live pnpm owns concrete workspace and virtual-store realization terms. Nix +prepared dependencies own prepared artifacts, native integrations, and hash +evidence. Buck2 owns its target and evidence representations. Those terms must +refer back to this ontology rather than redefine its identities or authorities. + +## Language + +### Materialization + +**Dependency Materialization** is the process that realizes declared dependency +inputs as a dependency graph and its derived outputs. + +**Materialization Root** is a concrete filesystem or build lifecycle boundary. +It realizes one active materialization profile at a time and owns any writable +realization state for that boundary. + +**Authoritative Materializer** is the role held by the sole mechanism allowed to +select or change the Package Instance targeted by a Dependency Edge. A faithful +restore may reproduce an already-selected edge, and repair may discard a whole +owned realization before reinvoking the materializer; neither may select a +replacement target. pnpm holds this role for a live pnpm root. + +**Materialization Profile** is a versioned, physical-root-location-independent +descriptor for equivalent immutable dependency work. It names normalized +topology and dependency inputs plus package-manager and toolchain policy. It +does not own mutable realization state, storage placement, repair, or garbage +collection. Prepared dependency and Buck2 evidence use its `profileKey` as a +compatibility boundary; a live root need not emit a separate profile artifact. + +### Dependency graph + +**Dependency Graph** is the package-instance nodes and dependency edges selected +by an authoritative materializer for one materialization root. + +**Package Instance** is one semantically distinct resolved node in a dependency +graph. Its opaque identity is selected by the authoritative materializer and +distinguishes every resolution-relevant discriminator, including version, peer +context, patch, injection, and platform. A pnpm package snapshot is one +concrete representation of this concept. + +**Dependency Edge** is a directed relation from a consuming package instance or +the materialization root to a dependency package instance. Its target identity +is authoritative graph data, not projection state. + +### Derived outputs + +**Dependency Data** is package content and graph data deterministically selected +from declared inputs under a materialization policy. It excludes mutable +package-manager state, projection state, and unclassified native or build +outputs. + +**Projection State** is reproducible, non-authoritative state derived from a +dependency graph, such as executable shims and local tool metadata. Projection +may observe dependency edges but may not write them. + +### Storage and lifecycle + +**Store Cache** is a disposable, package-manager-owned cache used while +materializing dependencies. It may contain immutable content-addressed package +files and mutable package-manager-derived lookup indexes. Neither facet is +authoritative Dependency Graph or Projection State. A Store Cache may be +shared by mutually trusted Materialization Roots without sharing their graphs. + +**Content-addressed Package Data** is the immutable byte layer whose identity is +derived from content. It may be reused across every compatible Materialization +Root. It is part of a Store Cache but does not include that cache's mutable +derived indexes. _Avoid_: CAS when referring to the whole pnpm Store Cache. + +**Reuse Scope** is the set of compatible consumers allowed to reuse equivalent +immutable data or deterministic work. It is an independent facet from Authority +Scope: widening reuse does not grant mutation or repair authority. + +**Authority Scope** is the smallest boundary within which one owner may mutate, +repair, or discard state without coordinating independent consumers. A live +Dependency Graph has Materialization-Root Authority Scope even when its package +data has host-user Reuse Scope. + +**Global Virtual Store** is pnpm's cross-project virtual-store realization. It +shares graph/topology realization state and is therefore distinct from sharing +a Store Cache or Content-addressed Package Data. _Avoid_: GVS as a synonym for +cache reuse or dependency identity. + +**Hermetic Dependency Artifact** is an immutable dependency-data and topology +result keyed by the complete declared graph, platform, package-manager policy, +and every other identity-affecting input. Construction is lifecycle-free and +atomic; consumers cannot mutate it. It may have broad Reuse Scope without +granting graph or repair authority, analogous to a Nix derivation result or a +build-system action result. A mutable pnpm Global Virtual Store is not such an +artifact. + +**Store Cache Lease** coordinates package-manager mutation with cache-owner +maintenance. Its shared **admission** mode permits concurrent materialization; +its exclusive **maintenance** mode excludes materialization while the Store +Cache is pruned. _Avoid_: install lock, global install lock. + +**Repair** is the restoration of a materialization root from declared inputs. +Repair may discard owned derived state and reinvoke the authoritative +materializer; it may not synthesize replacement dependency edges. + +## Structure + +The materialization pipeline is: + +```text +declared dependency inputs + Materialization Root + -> Authoritative Materializer + -> Dependency Graph + -> Dependency Data + -> Projection State + -> realization-specific native integration + -> materialization evidence +``` + +The weight-bearing relations are: + +| Subject | Relation | Object | +| ------------------------------ | ----------- | ------------------------------------ | +| Package Instance | `partOf` | Dependency Graph | +| Dependency Edge | `partOf` | Dependency Graph | +| Materialization Profile | `describes` | equivalent immutable dependency work | +| Dependency Graph | `partOf` | Materialization Root | +| Dependency Graph | `dependsOn` | Authoritative Materializer | +| Projection State | `dependsOn` | Dependency Graph | +| Materialization Root | `dependsOn` | Store Cache | +| Repair | `dependsOn` | Authoritative Materializer | +| Content-addressed Package Data | `partOf` | Store Cache | + +Store placement is a facet of a realization: local development may use a +host-scoped cache, CI may use a job-scoped cache, and Nix prepared dependencies +may use an independent builder cache. Placement does not change Materialization +Profile or Package Instance identity. + +## Flagged ambiguities + +- Use **Materialization Profile**, not bare **profile**, outside an immediately + established materialization-profile context. +- Use **Materialization Root** for the owner of mutable realization state; do + not call that state profile-owned. +- Qualify **identity** as **Materialization Profile identity** or **Package + Instance identity**. +- Qualify **authority** by the operation it governs. Materialization, repair, + and garbage collection are separate authorities unless explicitly proven to + coincide. +- Use **Store Cache** for the whole pnpm store. Do not use **shared store** to + imply shared Dependency Graph state, and do not use **content pool** for a + store that also contains pnpm-owned derived indexes. +- Use **CAS** only for a system with an explicit content-address/ownership/GC + contract. For pnpm, say **Content-addressed Package Data** for the byte layer + and **Store Cache** for the whole package-manager-owned cache. +- State **Reuse Scope** and **Authority Scope** independently; a broader reuse + scope is not evidence for broader graph, mutation, or repair authority. +- Use pnpm **package snapshot** only for the pnpm representation and **Package + Instance** for the cross-realization concept. diff --git a/context/dependency-materialization/requirements.md b/context/dependency-materialization/requirements.md index 1f9ddcdb5d..ebad817d95 100644 --- a/context/dependency-materialization/requirements.md +++ b/context/dependency-materialization/requirements.md @@ -6,17 +6,20 @@ These requirements define the dependency materialization contract used by effect-utils live pnpm tasks, Nix prepared dependency artifacts, CI jobs, and future Buck2 dependency evidence. +Canonical shared terms and relationships are defined in +[ontology.md](./ontology.md). + The contract is intentionally stricter than pnpm's default lifecycle model: pnpm resolves and links package contents, while executable projection, native -tooling, and repair are owned by effect-utils, Nix, or explicit profile -operations. +tooling, and repair are owned by effect-utils, Nix, or an explicit realization +authority. Subsystem requirements refine this root contract: - [01-live-pnpm](./01-live-pnpm/requirements.md) defines mutable worktree installs and topology ownership. - [02-projections](./02-projections/requirements.md) defines deterministic - executable and workspace projection. + executable and local metadata projection. - [03-nix-prepared-deps](./03-nix-prepared-deps/requirements.md) defines immutable Nix prepared dependency artifacts. - [04-store-authority](./04-store-authority/requirements.md) defines shared @@ -35,9 +38,10 @@ Subsystem requirements refine this root contract: - **A02 Nix authority:** Native tools, compiled native bindings, and runtime binaries that cannot be treated as pure package artifacts are supplied by Nix or explicit wrappers, not by pnpm lifecycle scripts. -- **A03 Profile consumers:** Live installs, Nix prepared dependency artifacts, - CI jobs, and Buck2 evidence may realize dependencies differently, but they use - one shared profile vocabulary for identity, policy, and authority. +- **A03 Materialization Profile consumers:** Prepared dependency artifacts and + Buck2 evidence use one immutable Materialization Profile vocabulary for + equivalent dependency inputs. Live installs use their declared inputs and + install contract directly. - **A04 Prepared artifacts are data:** Prepared pnpm dependency FODs are data artifacts. Build-time executable shims and native/build outputs are projection or build-layer concerns unless explicitly modeled as pure package @@ -57,7 +61,11 @@ Subsystem requirements refine this root contract: convergent version bump is preferred over maintaining parallel legacy prepared-deps policies. - **T04 Conservative repair:** Repair and GC commands may refuse to mutate when - they cannot prove the correct profile, platform, or shared-store authority. + they cannot identify the Materialization Root or prove the required + shared-content authority. +- **T05 Purity before reuse:** Work that cannot be derived deterministically + from declared inputs without lifecycle mutation remains outside the shared + reuse boundary, even when isolating or rebuilding it costs more time or disk. ## Requirements @@ -88,38 +96,46 @@ Subsystem requirements refine this root contract: execute package code. - **DMP-R08 Native output rejection:** Prepared dependency validation must reject unexpected compiled native outputs and known platform-specific package - directories unless the profile explicitly classifies them as pure package - data. + directories unless the Materialization Profile explicitly classifies them as + pure package data. ### Must make materialization identity explicit -- **DMP-R09 Profile identity:** Every dependency materialization root must have - a stable profile identity derived from topology, lockfile, package-manager - policy, toolchain inputs, platform trait, and projection namespace. -- **DMP-R10 Shared schema:** Live pnpm tasks, Nix prepared dependency artifacts, - CI jobs, and Buck2 evidence must use the same profile fields when describing - equivalent dependency work. -- **DMP-R11 Topology authority:** A profile must name the authoritative - workspace topology and install owner. Package-local or sibling-root install - state must not become authoritative implicitly. +- **DMP-R09 Materialization Profile identity:** When prepared dependency or + Buck2 evidence groups equivalent immutable dependency work, its stable + Materialization Profile identity must derive from topology, dependency + inputs, package-manager policy, and toolchain inputs, not physical root or + storage placement. +- **DMP-R10 Shared Materialization Profile schema:** Nix prepared dependency + artifacts and Buck2 evidence must use the same Materialization Profile fields + when describing equivalent dependency work. +- **DMP-R11 Topology and edge authority:** Each Materialization Root must name + authoritative workspace topology and one Authoritative Materializer. + Package-local or sibling-root state must not become authoritative implicitly. + Only the Authoritative Materializer may + select or change the Package Instance targeted by a Dependency Edge. A + faithful restore may reproduce an already-selected edge, and repair may + discard an owned realization and reinvoke the Authoritative Materializer; + neither may select a replacement target. ### Must preserve correctness under sharing -- **DMP-R12 Store trait:** A profile must declare one store trait such as - `ciJobLocal`, `darwinSplitCas`, `linuxSharedHardlink`, `isolated`, - `nixPreparedDeps`, or a proven future trait. -- **DMP-R13 Shared CAS safety:** Any content-addressed store shared by multiple - profiles may be swept only by an authority that can mark from every active - root that references the shared pool. -- **DMP-R14 Raw prune refusal:** A per-profile prune command must refuse to - mutate a shared files pool unless it is executing through the shared pool's - coordinated GC authority. +- **DMP-R12 Root-owned writable state:** Writable Dependency Graph, virtual + store, and Projection State must remain inside one Materialization Root. + Cross-root reusable state must satisfy DMP-R21; package-manager control-plane + metadata is not made pure merely by being disposable or non-authoritative. +- **DMP-R13 Store Cache safety:** Loss or eviction of a Store Cache must not + corrupt an already-materialized Dependency Graph. Cache completeness must + not be treated as root health or as an offline-readiness guarantee. +- **DMP-R14 Managed prune refusal:** An effect-utils-managed prune scoped to one + Materialization Root must not mutate a host-scoped Store Cache. - **DMP-R15 Repair determinism:** Repair commands must converge to the same - final dependency data and projection state for the same profile inputs. + final Dependency Graph, Dependency Data, and Projection State for the same + declared dependency inputs and materialization policy. ### Must be measured and verifiable -- **DMP-R16 Real-repo gates:** Changes to store traits, prepared artifact +- **DMP-R16 Real-repo gates:** Changes to storage sharing, prepared artifact purity, or projection ownership must be validated on at least one real downstream graph in addition to synthetic fixtures. - **DMP-R17 Negative lifecycle tests:** Test fixtures must prove that managed @@ -134,3 +150,29 @@ Subsystem requirements refine this root contract: - **DMP-R20 Verification architecture:** Changes to dependency materialization behavior must map to explicit fixture, proof, benchmark, or real-workload evidence before they become defaults. + +### Must maximize reuse inside a pure boundary + +- **DMP-R21 Pure reusable state:** Cross-root reusable state must be + deterministic, content-addressed or equivalently integrity-addressed, derived + only from declared inputs, and treated as immutable. Dependency lifecycle + scripts, ambient downloads, source compilation, and mutable native/build + outputs must not enter that reuse boundary. +- **DMP-R22 Safety-gated optimization:** Correct dependency identity, declared + graph authority, lifecycle purity, data safety, and bounded repair/failure + scope are hard admissibility constraints. Among designs that satisfy every + constraint, defaults must seek a non-dominated operating point across + physical bytes, repeated work, cold/warm latency, concurrency, and operational + complexity among evaluated admissible candidates rather than maximizing + shared mutable state. New admissible candidates remain open challengers. +- **DMP-R23 Reuse/authority separation:** Reuse Scope and Authority Scope must + remain independent. Equivalent immutable data or work may be reused as + broadly as evidence permits, while writable graph, projection, repair, and + lifecycle authority stays at the smallest independently recoverable scope. +- **DMP-R24 Hermetic topology reuse:** Repeated dependency resolution or + topology work must graduate into a cross-root reusable artifact only when + its complete identity is derived from declared inputs, construction is + lifecycle-free and atomic, consumers cannot mutate the result, and corruption + or eviction can be repaired without coordinating those consumers. Root-local + mutable realization is a compatibility boundary, not the long-term reuse + ideal. diff --git a/context/dependency-materialization/spec.md b/context/dependency-materialization/spec.md index aad51767d7..c6df9fa714 100644 --- a/context/dependency-materialization/spec.md +++ b/context/dependency-materialization/spec.md @@ -13,8 +13,7 @@ This spec defines: - the separation between dependency data, executable projections, and native build outputs; - the prepared dependency FOD purity boundary; -- the dependency materialization profile shape shared by live tasks, Nix, CI, - and Buck2 evidence; +- the prepared dependency profile shape shared by Nix and Buck2 evidence; - the repair, doctor, and benchmark gates for changing the policy. This spec does not define package-specific native integrations. Those belong in @@ -26,7 +25,7 @@ Subsystem specs refine this root model: ```text dependency-materialization/ 01-live-pnpm/ mutable worktree installs - 02-projections/ deterministic .bin and workspace projection + 02-projections/ deterministic executable and metadata projection 03-nix-prepared-deps/ immutable Nix prepared dependency artifacts 01-fod-hash-evidence/ cross-system FOD hash evidence 02-native-node-packages/ native package classification and grafting @@ -45,11 +44,12 @@ dependency-materialization/ | Dependency Data, Projections, And Native Outputs | DMP-R05, DMP-R06, DMP-R08 | | Prepared FOD Purity | DMP-R05, DMP-R08, DMP-R18 | | Pure Bin Projection | DMP-R06, DMP-R07, DMP-R17 | -| Profile Record | DMP-R09, DMP-R10, DMP-R11, DMP-R12 | -| Store Traits And Authorities | DMP-R12, DMP-R13, DMP-R14 | +| Prepared Profile Evidence | DMP-R09, DMP-R10 | +| Storage Ownership And Authorities | DMP-R12, DMP-R13, DMP-R14 | | Doctor And Repair | DMP-R15 | | Benchmark And Acceptance Gates | DMP-R16, DMP-R17, DMP-R18, DMP-R19 | | Verification Architecture | DMP-R16, DMP-R17, DMP-R18, DMP-R19, DMP-R20 | +| Pure Reuse Boundary | DMP-R21, DMP-R22, DMP-R23, DMP-R24 | ## Model @@ -59,12 +59,19 @@ canonical workspace inputs -> dependency data -> pure executable projection -> Nix/native wrapper integration - -> profile evidence and health reports + -> prepared profile evidence and live health reports ``` pnpm is responsible for resolving and linking package contents. It is not the authority for executing package lifecycle code in effect-utils-managed paths. +The long-term reuse unit is a hermetic dependency artifact keyed by every input +that can affect package identity and topology. It is constructed atomically, +never mutated by consumers, and may be reused as broadly as compatibility +permits. Today's root-local pnpm graph is the mutable compatibility boundary +used where pnpm cannot yet expose that artifact; it is not the architectural +ceiling. + ## Strict pnpm Install Policy Managed live installs and prepared dependency builds use the strict install @@ -167,58 +174,40 @@ The projection contract covers: - package aliases where the dependency name differs from the package name; - package-local bins for workspace package roots; - executable bit and shebang preservation on Unix; -- deterministic overwrite of stale shims owned by the profile. +- deterministic overwrite of stale shims owned by the Materialization Root. The projection does not cover package CLIs generated by postinstall. Those are native/build integration work. -## Profile Record +## Prepared Profile Evidence -Profile evidence is emitted as JSON. Producers may add fields, but the stable -core shape is: +Nix prepared-dependency and Buck2 evidence retain the existing `profileKey` +compatibility boundary. It describes immutable dependency work and excludes +live storage placement and operational authority: ```json { - "schema": "dependency-materialization-profile/v0", - "profileId": "pnpm:::::", - "topology": { - "ownerRoot": ".", - "workspaceDigest": "sha256:...", - "lockfileDigest": "sha256:...", - "manifestDigest": "sha256:..." - }, - "toolchain": { - "packageManager": "pnpm", - "packageManagerVersion": "11.x", - "nodeVersion": "24.x", - "policyDigest": "sha256:..." - }, - "policy": { - "lifecycleScripts": "ignored", - "optionalDependencies": "excluded | included", - "binProjection": "pure-manifest", - "nativeOutputs": "nix-or-pure-package-artifacts" + "kind": "dependency-materialization-profile", + "schemaVersion": 1, + "profileKey": "", + "identity": { + "installDir": ".", + "lockfilePath": "pnpm-lock.yaml", + "memberDirs": ["packages/app"], + "freshness": {}, + "policy": { + "packageManager": "pnpm", + "lockfileMode": "frozen", + "lifecycleScripts": "ignored" + } }, - "store": { - "trait": "ciJobLocal | darwinSplitCas | linuxSharedHardlink | isolated | nixPreparedDeps | frozenSeed", - "filesPoolId": "host-pnpm-v11-files", - "metadataNamespace": "profile-local", - "projectionNamespace": "profile-local" - }, - "authorities": { - "liveRepair": "nix-devenv", - "gc": "shared-pool-coordinator | profile-local | none", - "evidence": "buck2 | nix | devenv" - } + "depsHash": "sha256-..." } ``` -`profileId` is stable only for semantic dependency inputs. It does not include -machine-local checkout paths, temporary store paths, or CI job ids. - -The policy digest includes lifecycle policy, optional dependency policy, -prepared artifact normalization version, native-output classification, bin -projection policy, and store trait. +Live pnpm roots do not emit a second profile artifact. Their generated install +contract plus install and projection hashes are the evidence used to decide +whether installation is current. Nix prepared-deps producers also expose `fodHashRepairTargets` as evaluated package metadata. Each target derives from the same profile and `depsBuilds` @@ -226,37 +215,62 @@ hash declaration as the fixed-output derivation. Repair tools consume those targets to rebuild direct dependency artifacts and publish run evidence; package sources do not carry a second per-target witness file. -## Store Traits And Authorities - -| Trait | Intended use | Sharing model | GC authority | -| --------------------- | ----------------------------------- | ------------------------------------------------------- | ---------------------------------------- | -| `ciJobLocal` | CI jobs and isolated automation | job-local pnpm state | profile-local | -| `darwinSplitCas` | macOS local development | profile-local metadata/projection, shared package files | shared-pool coordinator | -| `linuxSharedHardlink` | Linux local development after proof | host-local store with hardlink import | shared-pool coordinator | -| `isolated` | fallback and reproduction | no sibling sharing | profile-local | -| `nixPreparedDeps` | Nix fixed-output dependency data | prepared tree in Nix store | Nix store | -| `frozenSeed` | future read-only seed | immutable seed plus writable projection | seed owner plus profile-local projection | - -Profile-local prune refuses to sweep a shared files pool. Shared-pool GC marks -from every active root before sweeping. +## Storage Ownership And Authorities + +| State | Scope | Mutation authority | +| --------------------------------------- | -------------------------- | ------------------------ | +| live dependency graph and virtual store | one Materialization Root | that root's pnpm install | +| live executable projection | one Materialization Root | pure projection task | +| pnpm Store Cache | host-local or CI-job-local | pnpm concurrency control | +| prepared dependency data | immutable Nix store output | Nix build | + +The current local-development pnpm compatibility realization shares one whole +Store Cache across mutually trusted roots owned by the same user. Because that +cache includes mutable pnpm indexes, it does not satisfy the pure reusable-state +boundary; the divergence is explicit in +[DELTA-001](./.delta/DELTA-001-whole-store-mutable-index.md). effect-utils +exposes no Materialization-Root repair or prune operation that sweeps that host +cache. Nix prepared-dependency production remains an independent immutable path +and does not consume the live host cache. + +## Pure Reuse Boundary + +Dependency work becomes eligible for cross-root reuse only after it is derived +from declared inputs without lifecycle execution and has an immutable, +integrity-addressed identity. Package contents that satisfy that boundary may +be reused across every compatible root. Mutable package-manager indexes, graph +edges, projections, repair state, ambient downloads, lifecycle output, and +unclassified native/build output do not become reusable merely because they +can be placed in a shared directory or discarded after failure. + +The optimization order is lexicographic: + +1. preserve dependency identity, declared authority, purity, data safety, and + independently bounded repair; +2. within that feasible set, minimize physical bytes and repeated work; +3. select a non-dominated latency/concurrency/complexity operating point among + evaluated admissible candidates while leaving the challenger set open. + +This is the same structural property exploited by Nix and hermetic build graphs: +make the reusable unit pure and addressable first, then widen reuse without +widening mutation authority. ## Doctor And Repair Doctor checks: -- profile identity matches the current topology and policy; +- the install contract and cached state match the current topology and policy; - dependency data is present; -- shared package files are present for the profile; -- prepared artifact scans pass for the selected profile; +- dependency data referenced by the current graph is present; +- prepared artifact scans pass for the selected inputs; - expected `.bin` projection entries exist and point at package data; -- no lifecycle sentinel output exists for test profiles. +- no lifecycle sentinel output exists in verification fixtures. Repair may: - rerun pure pnpm install with scripts disabled; - recreate `.bin` projection; - restore prepared data from Nix; -- route shared-pool GC or repair to the coordinated authority. Repair may not: @@ -276,13 +290,10 @@ A materialization policy change is accepted only when it proves: leaked pnpm state; - fixed-output hashes are measured per covered system, with missing systems marked pending rather than silently collapsed; -- host-wide bytes, file counts, cold install, warm install, offline reinstall, - and repair times are recorded. +- host-wide bytes, file counts, cold install, warm install, and concurrent + install are recorded; offline reinstall and repair timing are required only + when those capabilities are explicitly claimed. The [verification subsystem](./07-verification/spec.md) owns the evidence matrix, proof tiers, benchmark record shape, and regression-gate routing for these acceptance gates. - -## Open Design Questions - -No open design questions remain for this milestone. diff --git a/devenv.nix b/devenv.nix index 2df135d571..77c8a3a6d6 100644 --- a/devenv.nix +++ b/devenv.nix @@ -426,9 +426,10 @@ in workspaceFilter = true; }) packagesWithNetlifyPreview; }) - (taskModules.workflow-report { - ciToolsBin = "${ciToolsSourceCli}/bin/ci-tools"; - }) + # Workflow reports run as standalone CI control-plane steps, including when + # a deploy is skipped. Use the hermetic package instead of relying on an + # ambient source-workspace node_modules projection. + (taskModules.workflow-report { }) (taskModules.lint-oxc { oxlintPkg = oxlintWithPlugins; lintPaths = [ @@ -588,18 +589,12 @@ in local package_name="$1" local package_path="$2" local rel_path="$package_name" - local gvs_links_dir local search_roots=(node_modules) if [[ "$package_name" == @*/* ]]; then rel_path="$(dirname "$package_name")/$(basename "$package_name")" fi - gvs_links_dir="$(resolve_gvs_links_dir)" - if [[ -n "$gvs_links_dir" && -d "$gvs_links_dir" ]]; then - search_roots+=("$gvs_links_dir") - fi - find "''${search_roots[@]}" \ -path "*/node_modules/$rel_path" \ -exec sh -c 'package_path="$1"; shift; for target do rm -rf "$target"; ln -s "$package_path" "$target"; done' sh "$package_path" {} + @@ -715,7 +710,18 @@ in ''; }; - tasks."check:all".after = [ "cargo:check" ]; + tasks."dependency-materialization:evidence:check" = { + description = "Validate committed dependency-materialization benchmark and host-capability evidence"; + exec = trace.exec "dependency-materialization:evidence:check" '' + ${pkgs.nodejs}/bin/node \ + context/dependency-materialization/07-verification/evidence/validate-storage-sharing-default.mjs + ''; + }; + + tasks."check:all".after = [ + "cargo:check" + "dependency-materialization:evidence:check" + ]; # Keep git-hook installation out of the shell-entry path. # If needed, install with `devenv tasks run devenv:git-hooks:install`. diff --git a/flake.nix b/flake.nix index 5414a55577..728b223e97 100644 --- a/flake.nix +++ b/flake.nix @@ -329,12 +329,6 @@ # Usage: effectUtils.lib.mkPnpm { inherit pkgs; } lib.mkPnpm = { pkgs }: import ./nix/pnpm.nix { inherit pkgs; }; - # Cross-repo effect-utils source relink install command, with the pnpm - # store-dir resolved from the CI-exported env so the relink shares one - # store with the root install (avoids a two-store split in deploy jobs). - # Usage: effectUtils.lib.mkEffectUtilsInstall { root = config.devenv.root; } - lib.mkEffectUtilsInstall = import ./nix/devenv-modules/tasks/lib/effect-utils-install.nix; - # Note: mkSourceCli is internal-only (not exported). # For consuming CLIs from other repos, use: # effectUtils.packages.${system}.genie diff --git a/genie/ci-workflow/setup.ts b/genie/ci-workflow/setup.ts index 48180ef7d8..96186094cb 100644 --- a/genie/ci-workflow/setup.ts +++ b/genie/ci-workflow/setup.ts @@ -351,7 +351,8 @@ echo "Pinned devenv rev: $DEVENV_REV"`, /** * Export the canonical CI pnpm paths once so every later shell step shares the - * same writable store and the same workspace-relative GVS projection. + * same job-local home and content store. Writable virtual topology stays under + * the workspace root. */ export const pnpmStateSetupStep = { name: 'Isolate pnpm state', diff --git a/genie/ci-workflow/shared.ts b/genie/ci-workflow/shared.ts index 3948dea9e7..88939e355a 100644 --- a/genie/ci-workflow/shared.ts +++ b/genie/ci-workflow/shared.ts @@ -272,30 +272,20 @@ export const withAppendedNixConfig = ({ export const dollar = '$' /** - * Keep pnpm's hot mutable content isolated per job while still allowing cache reuse across runs. - * - * In the pnpm 11 + GVS configuration we use today, the effective hot state lives - * under `PNPM_HOME`, not `PNPM_STORE_DIR`. `PNPM_HOME` must stay - * workspace-relative because the GVS links embed absolute paths and those need - * to stay valid for relocatable artifacts like `vercel deploy --prebuilt`. + * Keep pnpm's auxiliary home state isolated per job. */ export const jobLocalPnpmHome = '${{ github.workspace }}/.pnpm-home' /** * Keep pnpm's auxiliary mutable store content isolated per job. * - * We still wire `PNPM_STORE_DIR` explicitly for pnpm, but the primary CI cache - * target is `PNPM_HOME` because that is where pnpm 11 GVS keeps the reusable - * links and metadata. + * The writable virtual topology remains under the workspace's + * `node_modules/.pnpm`; the store carries content and auxiliary metadata. */ export const jobLocalPnpmStore = '${{ runner.temp }}/pnpm-store/${{ github.job }}' /** - * Canonical pnpm CI state surface for pnpm 11 + GVS on self-hosted runners. - * - * `PNPM_HOME` carries the hot reusable links and metadata, while the - * auxiliary mutable store content still lives under `PNPM_STORE_DIR`. The - * supported cache contract restores both together under one exact key. + * Canonical job-local pnpm CI state surface on self-hosted runners. */ export const jobLocalPnpmStatePaths = [jobLocalPnpmHome, jobLocalPnpmStore].join('\n') diff --git a/genie/external.ts b/genie/external.ts index cb15db157d..4bee455054 100644 --- a/genie/external.ts +++ b/genie/external.ts @@ -141,6 +141,74 @@ export type { export { nativeDependencyPolicy } export type { NativeDependencyPolicyEntry } +/** Storage portion of effect-utils/pnpm-install-contract schema v2. */ +export interface PnpmInstallStorageContractV2 { + readonly storeContract: { + readonly owner: 'pnpm' + readonly layoutVersion: 'v11' + readonly localDevelopment: { + readonly scope: 'host-user' + readonly trustBoundary: 'same-os-user' + readonly defaultPath: '~/.local/share/pnpm/store-shared-v1' + readonly pathOverrideEnvironmentVariable: 'PNPM_SHARED_STORE_DIR' + readonly contentAddressedFiles: 'shared' + readonly derivedIndex: 'shared-pnpm-owned' + } + readonly ci: { + readonly scope: 'job' + } + readonly virtualStore: { + readonly scope: 'materialization-root' + readonly path: 'node_modules/.pnpm' + readonly global: false + } + } + readonly packageImportMethod: { + readonly live: { + readonly method: 'auto' + readonly owner: 'pnpm' + readonly linuxSameDeviceRequired: true + } + readonly nixPreparedDependencies: { + readonly scope: 'independent-builder-policy' + } + } +} + +/** Shared live/Nix storage authority for every megarepo install contract. */ +export const pnpmInstallStorageContractV2 = { + storeContract: { + owner: 'pnpm', + layoutVersion: 'v11', + localDevelopment: { + scope: 'host-user', + trustBoundary: 'same-os-user', + defaultPath: '~/.local/share/pnpm/store-shared-v1', + pathOverrideEnvironmentVariable: 'PNPM_SHARED_STORE_DIR', + contentAddressedFiles: 'shared', + derivedIndex: 'shared-pnpm-owned', + }, + ci: { + scope: 'job', + }, + virtualStore: { + scope: 'materialization-root', + path: 'node_modules/.pnpm', + global: false, + }, + }, + packageImportMethod: { + live: { + method: 'auto', + owner: 'pnpm', + linuxSameDeviceRequired: true, + }, + nixPreparedDependencies: { + scope: 'independent-builder-policy', + }, + }, +} satisfies PnpmInstallStorageContractV2 + // ============================================================================= // Shared label catalog (consumed by per-repo `.github/labels.json.genie.ts`) // ============================================================================= @@ -343,11 +411,10 @@ const deniedLifecycleBuilds = Object.fromEntries( * This is the SSOT for pnpm strictness/layout policy. Every megarepo * root workspace should spread this so policy stays consistent. * - * `enableGlobalVirtualStore` ensures identity convergence: equivalent - * dependency graphs across standalone and composed topologies resolve to - * the same physical instance via pnpm's global content-addressed store. - * This eliminates duplicate-instance problems (TypeScript type identity, - * JS runtime singletons) when consuming cross-repo packages via `link:`. + * The virtual dependency graph is always root-local. Cross-root reuse is + * limited to pnpm's Store Cache inside one same-user trust boundary; composed + * runtime identity is established by the composed workspace topology, never by + * shared writable graph state. */ export const commonPnpmPolicySettings = { dedupePeerDependents: true as const, @@ -363,9 +430,7 @@ export const commonPnpmPolicySettings = { vitest: '>=4.0.0', }, }, - enableGlobalVirtualStore: true as const, - storeDir: '.devenv/pnpm-store-pure-v1', - packageImportMethod: 'clone-or-copy' as const, + packageImportMethod: 'auto' as const, sideEffectsCache: false as const, verifyStoreIntegrity: true as const, strictStorePkgContentCheck: true as const, diff --git a/genie/internal.ts b/genie/internal.ts index 6f8cc4b21c..b3bc479310 100644 --- a/genie/internal.ts +++ b/genie/internal.ts @@ -150,8 +150,7 @@ export const commonPnpmWorkspaceData = { packageExtensions: { ...commonPnpmPolicySettings.packageExtensions, // Storybook loads the configured framework preset dynamically from the - // storybook package. Under pnpm's global virtual store, that import cannot - // see the workspace package's dev dependency unless the edge is explicit. + // storybook package, so the dependency edge must be explicit. storybook: { dependencies: { '@storybook/react-vite': '10.4.6', diff --git a/genie/packages.ts b/genie/packages.ts index 95a2d4320a..e2d82bea97 100644 --- a/genie/packages.ts +++ b/genie/packages.ts @@ -48,10 +48,8 @@ export type InternalPackageName = (typeof internalPackages)[number] /** * Generate catalog entries for all internal packages. - * Using `workspace:^` (not `workspace:*`) so pnpm resolves the actual version - * from package.json. This is critical for GVS: with `workspace:*`, pnpm stores - * workspace packages with `undefined` as version in the global link store, - * breaking TypeScript resolution through GVS real paths. + * Using `workspace:^` (not `workspace:*`) so pnpm resolves and records the + * actual version from package.json in every standalone and composed topology. */ export const internalPackageCatalogEntries = Object.fromEntries( internalPackages.map((name) => [`@overeng/${name}`, 'workspace:^'] as const), diff --git a/nix/devenv-modules/tasks/README.md b/nix/devenv-modules/tasks/README.md index 21b29cdc1e..e0dd543728 100644 --- a/nix/devenv-modules/tasks/README.md +++ b/nix/devenv-modules/tasks/README.md @@ -38,11 +38,15 @@ imports = [ - `megarepo.nix` - Megarepo workspace tasks - `nix-cli.nix` - Nix CLI build/check tasks - `pnpm.nix` - pnpm install tasks - - Default live-worktree store namespace is `.devenv/pnpm-store-pure-v1`. - - Local development may share only pnpm `v11/files`; mutable metadata, - GVS `links`, `projects`, temp state, and CI state remain local/job-local. - - Managed installs enforce mutation-isolating imports and reject writable - hardlink or side-effects-cache overrides. + - Local development shares one complete pnpm Store Cache between trusted + roots of the same OS user; CI uses a job-local Store Cache. + - Dependency graphs, `node_modules/.pnpm`, projections, and repair remain + Materialization-Root-owned. + - Managed installs use pnpm's `auto` import policy and reject cross-device + Linux storage before materialization. + - `pnpm:store:migrate-legacy` explicitly replaces only the recognized + historical `v11/files` bridge under the exclusive cache lease; normal + installs and unknown bridges fail closed. - Frozen installs use the current guarded pnpm runtime, while `pnpm:update` uses a separate pnpm 11.5.1 lock mutator. Root updates generate projections with validation deferred, repair the lock, then require `genie --check`; diff --git a/nix/devenv-modules/tasks/lib/effect-utils-install.nix b/nix/devenv-modules/tasks/lib/effect-utils-install.nix deleted file mode 100644 index 9a20e4a32c..0000000000 --- a/nix/devenv-modules/tasks/lib/effect-utils-install.nix +++ /dev/null @@ -1,21 +0,0 @@ -# Shared pnpm install command for the cross-repo effect-utils source relink. -# -# Repos that compose effect-utils as a linked pnpm source workspace must -# pre-warm `repos/effect-utils`'s own node_modules so TypeScript resolves -# imports through the cross-repo symlinks. Centralizing the command here keeps -# the pnpm store-dir resolution in one place: it is sourced from the -# CI-exported env (PNPM_CONFIG_STORE_DIR / PNPM_STORE_DIR) so the relink shares -# one store with the root install, falling back to the workspace store locally. -# -# A hardcoded CLI `--config.store-dir` has the highest pnpm precedence and would -# override the CI env, splitting a single deploy job across two stores and -# desyncing the root node_modules projection (the `next: command not found` -# class of failure). `--force` performs the GVS relink. `--frozen-lockfile` is -# explicit (not relying on pnpm's CI-only default for frozen) so a drifted -# effect-utils lockfile fails loudly in both local and CI runs rather than being -# silently rewritten. -{ - root, - dir ? "repos/effect-utils", -}: -''DEVENV_TASK_PASSTHROUGH=1 pnpm install --force --frozen-lockfile --ignore-scripts --config.confirmModulesPurge=false --config.store-dir="''${PNPM_CONFIG_STORE_DIR:-''${PNPM_STORE_DIR:-${root}/.devenv/pnpm-store-pure-v1}}" --dir ${dir}'' diff --git a/nix/devenv-modules/tasks/shared/bun.nix b/nix/devenv-modules/tasks/shared/bun.nix index 5cab1c5229..3a0ad68336 100644 --- a/nix/devenv-modules/tasks/shared/bun.nix +++ b/nix/devenv-modules/tasks/shared/bun.nix @@ -9,7 +9,7 @@ # Why we want bun: # - Significantly faster installs (when not hitting bugs) # - bun's file: protocol works like pnpm's link: (symlinks with own deps) -# - No need for enableGlobalVirtualStore workaround +# - Root-local dependency topology by default # # See: context/workarounds/bun-issues.md # diff --git a/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs b/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs index c4b9e0591b..f47104487c 100644 --- a/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs +++ b/nix/devenv-modules/tasks/shared/check-node-modules-projection-health.cjs @@ -13,11 +13,33 @@ const moduleDirs = (process.env.NODE_MODULES_DIRS || '') const existingModuleDirs = moduleDirs.filter((value) => fs.existsSync(value)) +const rootModulesYamlPath = process.env.PNPM_ROOT_MODULES_YAML || 'node_modules/.modules.yaml' +const rootNodeModulesPath = path.resolve(moduleDirs[0] || path.dirname(rootModulesYamlPath)) +const rootNodeModulesDir = fs.existsSync(rootNodeModulesPath) + ? fs.realpathSync(rootNodeModulesPath) + : rootNodeModulesPath +const rootVirtualStoreDir = path.join(rootNodeModulesDir, '.pnpm') + +const isWithin = (parentPath, childPath) => { + const relativePath = path.relative(parentPath, childPath) + return ( + relativePath === '' || + (!relativePath.startsWith(`..${path.sep}`) && + relativePath !== '..' && + !path.isAbsolute(relativePath)) + ) +} + +const isPnpmPackageInstance = (packageDir) => + packageDir.includes(`${path.sep}node_modules${path.sep}.pnpm${path.sep}`) + const collectProjectionEntryPaths = (nodeModulesDir) => { const result = [] for (const entry of fs.readdirSync(nodeModulesDir, { withFileTypes: true })) { + if (entry.name === '.bin' || entry.name === '.pnpm') continue + const entryPath = path.join(nodeModulesDir, entry.name) - if (entry.isDirectory()) { + if (entry.name.startsWith('@') && entry.isDirectory()) { for (const childEntry of fs.readdirSync(entryPath, { withFileTypes: true })) { result.push(path.join(entryPath, childEntry.name)) } @@ -29,6 +51,35 @@ const collectProjectionEntryPaths = (nodeModulesDir) => { return result.sort() } +const collectVirtualStoreDependencyEdgePaths = (virtualStoreDir) => { + if (!fs.existsSync(virtualStoreDir)) return [] + + const result = [] + for (const locatorEntry of fs.readdirSync(virtualStoreDir, { withFileTypes: true })) { + if (!locatorEntry.isDirectory()) continue + + const locatorNodeModulesDir = path.join(virtualStoreDir, locatorEntry.name, 'node_modules') + if (!fs.existsSync(locatorNodeModulesDir)) continue + + for (const packageEntryPath of collectProjectionEntryPaths(locatorNodeModulesDir)) { + const packageEntryStat = fs.lstatSync(packageEntryPath) + if (packageEntryStat.isSymbolicLink()) { + result.push(packageEntryPath) + continue + } + if (!packageEntryStat.isDirectory()) continue + + const nestedNodeModulesDir = path.join(packageEntryPath, 'node_modules') + if (!fs.existsSync(nestedNodeModulesDir)) continue + for (const dependencyEntryPath of collectProjectionEntryPaths(nestedNodeModulesDir)) { + if (fs.lstatSync(dependencyEntryPath).isSymbolicLink()) result.push(dependencyEntryPath) + } + } + } + + return result.sort() +} + const collectHealthEntryPaths = (nodeModulesDir) => { const result = [] for (const entry of fs.readdirSync(nodeModulesDir, { withFileTypes: true })) { @@ -126,6 +177,11 @@ const packageTargetIsShipped = ({ includedFiles, target }) => { if (!target.startsWith('./')) return false if (target.includes('*')) return false + // npm includes package contents by default when `files` is omitted. An + // explicit non-empty allowlist is the only case where targets can be + // classified as outside the published package from manifest data alone. + if (!Array.isArray(includedFiles) || includedFiles.length === 0) return true + const relativeTarget = target.slice(2) return includedFiles.some( (file) => file === relativeTarget || relativeTarget.startsWith(`${file}/`), @@ -133,7 +189,7 @@ const packageTargetIsShipped = ({ includedFiles, target }) => { } const verifyPackageContent = ({ pkg, packageDir, entryPath, failures }) => { - if (!packageDir.includes('/v11/links/')) return + if (!packageDir.includes('/node_modules/.pnpm/')) return const includedFiles = Array.isArray(pkg.files) ? pkg.files.filter((file) => typeof file === 'string' && !file.startsWith('!')) @@ -174,7 +230,43 @@ const runProjectionHash = () => { hash.update('\n') } - appendLine(`gvs-links-dir ${process.env.PNPM_GVS_LINKS_DIR || ''}`) + const appendSymlinkEvidence = (entryPath) => { + let target = '' + try { + target = fs.readlinkSync(entryPath) + } catch {} + + appendLine( + `${fs.existsSync(entryPath) ? 'link' : 'broken-link'} ${entryPath} -> ${target}`, + ) + } + + const appendPackageContentEvidence = (entryPath) => { + let packageDir + try { + packageDir = fs.realpathSync(entryPath) + } catch { + return + } + + const packageJsonPath = path.join(packageDir, 'package.json') + if (!fs.existsSync(packageJsonPath)) return + const packageJsonBytes = fs.readFileSync(packageJsonPath) + const pkg = JSON.parse(packageJsonBytes.toString('utf8')) + appendLine( + `package-json ${entryPath} ${crypto.createHash('sha256').update(packageJsonBytes).digest('hex')}`, + ) + + const includedFiles = Array.isArray(pkg.files) ? pkg.files : undefined + const runtimeTargets = [pkg.main, ...collectRootRuntimeExportTargets(pkg.exports)] + .filter((target) => typeof target === 'string') + .filter((target) => packageTargetIsShipped({ includedFiles, target })) + for (const target of [...new Set(runtimeTargets)].sort()) { + appendLine( + `runtime-target ${entryPath} ${target} ${targetExistsWithNodeResolution(packageDir, target) ? 'present' : 'missing'}`, + ) + } + } for (const nodeModulesDir of moduleDirs) { if (fs.existsSync(nodeModulesDir) && fs.statSync(nodeModulesDir).isDirectory()) { @@ -194,20 +286,15 @@ const runProjectionHash = () => { if (!stat.isSymbolicLink()) continue - let target = '' - try { - target = fs.readlinkSync(entryPath) - } catch {} - - if (fs.existsSync(entryPath)) { - appendLine(`link ${entryPath} -> ${target}`) - } else { - appendLine(`broken-link ${entryPath} -> ${target}`) - } + appendSymlinkEvidence(entryPath) + appendPackageContentEvidence(entryPath) } } - const rootModulesYamlPath = process.env.PNPM_ROOT_MODULES_YAML || 'node_modules/.modules.yaml' + for (const edgePath of collectVirtualStoreDependencyEdgePaths(rootVirtualStoreDir)) { + appendSymlinkEvidence(edgePath) + } + if (fs.existsSync(rootModulesYamlPath)) { appendLine( `modules-yaml ${crypto @@ -224,6 +311,7 @@ const runProjectionHash = () => { const runHealthCheck = () => { const dependencyProjectionFailures = [] + const packageIdentityFailures = [] const packageContentFailures = [] for (const nodeModulesDir of existingModuleDirs) { @@ -248,6 +336,14 @@ const runHealthCheck = () => { if (!fs.existsSync(packageJsonPath)) continue const pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf8')) + + if (isPnpmPackageInstance(realPath) && !isWithin(rootVirtualStoreDir, realPath)) { + packageIdentityFailures.push( + `${pkg.name ?? entryPath} -> ${realPath} (expected within ${rootVirtualStoreDir})`, + ) + continue + } + verifyPackageContent({ pkg, packageDir: realPath, @@ -255,21 +351,38 @@ const runHealthCheck = () => { failures: packageContentFailures, }) - if (!realPath.includes('/v11/links/')) continue + if (!realPath.includes('/node_modules/.pnpm/')) continue - const dependencyNames = Object.keys(pkg.dependencies ?? {}) + const requiredDependencyNames = new Set(Object.keys(pkg.dependencies ?? {})) + const dependencyNames = [ + ...new Set([...requiredDependencyNames, ...Object.keys(pkg.peerDependencies ?? {})]), + ] if (dependencyNames.length === 0) continue const requireFromPkg = createRequire(packageJsonPath) for (const dependencyName of dependencyNames) { + const dependencyRoot = resolveDependencyPackageRoot({ + requireFromPkg, + dependencyName, + }) + if (dependencyRoot === undefined) { + if (requiredDependencyNames.has(dependencyName)) { + dependencyProjectionFailures.push( + `${pkg.name ?? entryPath} -> ${dependencyName} (from ${nodeModulesDir})`, + ) + } + continue + } + + if (dependencyRoot === 'builtin') continue + + const dependencyRealPath = fs.realpathSync(dependencyRoot) if ( - resolveDependencyPackageRoot({ - requireFromPkg, - dependencyName, - }) === undefined + isPnpmPackageInstance(dependencyRealPath) && + !isWithin(rootVirtualStoreDir, dependencyRealPath) ) { - dependencyProjectionFailures.push( - `${pkg.name ?? entryPath} -> ${dependencyName} (from ${nodeModulesDir})`, + packageIdentityFailures.push( + `${pkg.name ?? entryPath} -> ${dependencyName} -> ${dependencyRealPath} (expected within ${rootVirtualStoreDir})`, ) } } @@ -279,11 +392,18 @@ const runHealthCheck = () => { for (const failure of dependencyProjectionFailures) { console.error(`[pnpm] Missing dependency projection: ${failure}`) } + for (const failure of packageIdentityFailures) { + console.error(`[pnpm] Foreign dependency package instance: ${failure}`) + } for (const failure of packageContentFailures) { console.error(`[pnpm] Missing package content: ${failure}`) } - if (dependencyProjectionFailures.length > 0 || packageContentFailures.length > 0) { + if ( + dependencyProjectionFailures.length > 0 || + packageIdentityFailures.length > 0 || + packageContentFailures.length > 0 + ) { process.exit(1) } } diff --git a/nix/devenv-modules/tasks/shared/pnpm-task-helpers.sh b/nix/devenv-modules/tasks/shared/pnpm-task-helpers.sh index 8afe18d9b7..b85a31c9c3 100644 --- a/nix/devenv-modules/tasks/shared/pnpm-task-helpers.sh +++ b/nix/devenv-modules/tasks/shared/pnpm-task-helpers.sh @@ -12,6 +12,182 @@ ensure_local_pnpm_home_default() { fi } +configure_pnpm_storage() { + local node_bin="$1" + local materialization_root="$2" + local job_local_store="$3" + local host_is_linux="$4" + local store_dir + local package_import_method="auto" + + if [ -n "${CI:-}" ]; then + store_dir="$job_local_store" + else + if [ -n "${PNPM_SHARED_STORE_DIR:-}" ]; then + store_dir="$PNPM_SHARED_STORE_DIR" + else + store_dir="$HOME/.local/share/pnpm/store-shared-v1" + fi + + local store_version_dir="$store_dir/v11" + local files_path="$store_version_dir/files" + + if [ -L "$store_version_dir" ]; then + echo "[pnpm] Refusing external pnpm Store Cache version bridge at $store_version_dir; discard and recreate the disposable $store_dir cache" >&2 + return 1 + fi + + if [ -L "$files_path" ]; then + echo "[pnpm] Refusing external pnpm Store Cache bridge at $files_path; discard and recreate the disposable $store_dir cache" >&2 + return 1 + fi + + mkdir -p "$files_path" + + if ! "$node_bin" - "$store_dir" "$files_path" <<'EOF' +const fs = require('node:fs') +const path = require('node:path') + +const [storeDir, filesPath] = process.argv.slice(2) +const realStoreDir = fs.realpathSync(storeDir) +const realFilesPath = fs.realpathSync(filesPath) +const relative = path.relative(realStoreDir, realFilesPath) + +if (relative.startsWith('..' + path.sep) || path.isAbsolute(relative)) { + console.error( + `[pnpm] Refusing pnpm Store Cache files outside selected store: store=${realStoreDir} files=${realFilesPath}`, + ) + process.exit(1) +} +EOF + then + return 1 + fi + + if [ "$host_is_linux" = true ]; then + "$node_bin" - "$materialization_root" "$files_path" <<'EOF' +const fs = require('node:fs') + +const [materializationRoot, filesPath] = process.argv.slice(2) +const rootDevice = fs.statSync(materializationRoot).dev +const filesDevice = fs.statSync(filesPath).dev + +if (rootDevice !== filesDevice) { + console.error( + `[pnpm] Zero-copy pnpm storage requires one filesystem: root=${materializationRoot} store-files=${filesPath}`, + ) + process.exit(1) +} +EOF + fi + + fi + + export PNPM_STORE_DIR="$store_dir" + export PNPM_CONFIG_STORE_DIR="$store_dir" + export npm_config_store_dir="$store_dir" + export PNPM_PACKAGE_IMPORT_METHOD="$package_import_method" +} + +acquire_pnpm_store_cache_lease() { + local flock_bin="$1" + local mode="$2" + local store_dir="$3" + local timeout_seconds="${4:-600}" + local lockfile="$store_dir/.effect-utils-pnpm-store-cache-maintenance.lock" + local flock_mode + + case "$mode" in + shared) flock_mode="--shared" ;; + exclusive) flock_mode="--exclusive" ;; + *) + echo "[pnpm] Invalid Store Cache lease mode: $mode" >&2 + return 2 + ;; + esac + + mkdir -p "$store_dir" + exec 202>"$lockfile" + if ! "$flock_bin" "$flock_mode" -w "$timeout_seconds" 202; then + echo "[pnpm] Store Cache $mode lease timeout after ${timeout_seconds}s: $lockfile" >&2 + return 1 + fi +} + +migrate_legacy_pnpm_store_cache() { + local store_dir="$1" + local expected_legacy_files="$2" + local store_version_dir="$store_dir/v11" + local files_path="$store_version_dir/files" + + if [ -L "$store_version_dir" ]; then + echo "[pnpm] Refusing to migrate a linked Store Cache version directory: $store_version_dir" >&2 + return 1 + fi + if [ ! -L "$files_path" ]; then + if [ -d "$files_path" ]; then + echo "[pnpm] Store Cache is already self-contained: $store_dir" + return 0 + fi + echo "[pnpm] No recognized legacy Store Cache bridge exists at $files_path" >&2 + return 1 + fi + + local actual_legacy_files + local expected_legacy_real + actual_legacy_files="$(readlink -f "$files_path")" + expected_legacy_real="$(readlink -f "$expected_legacy_files")" + if [ "$actual_legacy_files" != "$expected_legacy_real" ]; then + echo "[pnpm] Refusing unknown legacy Store Cache bridge: expected=$expected_legacy_real actual=$actual_legacy_files" >&2 + return 1 + fi + + # The caller holds the exclusive Store Cache lease. Reset only pnpm's + # disposable versioned metadata in place: the store root and maintenance-lock + # inode stay stable, and the historical external content pool is untouched. + find "$store_version_dir" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} + + mkdir -p "$files_path" + echo "[pnpm] Migrated legacy Store Cache bridge to a self-contained cache: $store_dir" +} + +assert_pnpm_storage_capacity() { + local node_bin="$1" + local store_dir="$2" + local materialization_root="$3" + + if [ -n "${CI:-}" ]; then + return 0 + fi + + local min_free_kib="${PNPM_MIN_FREE_KIB:-2097152}" + local boundary + local available_kib + + # pnpm `auto` may place cache files and a Materialization Root on distinct + # devices (notably Darwin clone/copy fallbacks). Check both physical write + # boundaries, but check a shared device only once. + while IFS= read -r boundary; do + available_kib="$(df -Pk "$boundary" | awk 'NR == 2 { print $4 }')" + if [ -z "$available_kib" ] || [ "$available_kib" -lt "$min_free_kib" ]; then + echo "[pnpm] Refusing materialization at $boundary with ${available_kib:-unknown} KiB free; require at least $min_free_kib KiB" >&2 + return 1 + fi + done < <("$node_bin" - "$store_dir" "$materialization_root" <<'EOF' +const fs = require('node:fs') + +const paths = process.argv.slice(2) +const seenDevices = new Set() +for (const path of paths) { + const device = fs.statSync(path).dev.toString() + if (!seenDevices.has(device)) { + seenDevices.add(device) + process.stdout.write(`${path}\n`) + } +} +EOF + ) +} + emit_dir_state() { local dir="$1" @@ -38,52 +214,6 @@ emit_dir_state() { done } -resolve_gvs_links_dir() { - # pnpm 11 stores the GVS links under the effective store-dir. Prefer the - # explicit store setting when tasks share storage across isolated PNPM_HOME - # directories. - if [ -n "${npm_config_store_dir:-}" ]; then - printf '%s\n' "${npm_config_store_dir}/v11/links" - elif [ -n "${PNPM_STORE_DIR:-}" ]; then - printf '%s\n' "${PNPM_STORE_DIR}/v11/links" - elif [ -n "${PNPM_HOME:-}" ]; then - printf '%s\n' "${PNPM_HOME}/store/v11/links" - elif [ -n "${XDG_DATA_HOME:-}" ] && [ -d "${XDG_DATA_HOME}/pnpm/store/v11" ]; then - printf '%s\n' "${XDG_DATA_HOME}/pnpm/store/v11/links" - elif [ -d "$HOME/.local/share/pnpm/store/v11" ]; then - printf '%s\n' "$HOME/.local/share/pnpm/store/v11/links" - elif [ -d "$HOME/Library/pnpm/store/v11" ]; then - printf '%s\n' "$HOME/Library/pnpm/store/v11/links" - fi -} - -cache_fingerprint() { - local workspace_state_hash="$1" - local gvs_links_dir="$2" - - # pnpm 11 bakes absolute paths into the live GVS projection, so two installs - # with identical manifests but different projection roots are not equivalent. - { - printf '%s\n' "$workspace_state_hash" - printf '%s\n' "$gvs_links_dir" - } | compute_hash -} - -resolve_pnpm_install_contract_file() { - local dir="${1:-$PWD}" - - while [ "$dir" != "/" ]; do - if [ -f "$dir/pnpm-install-contract.json" ]; then - printf '%s\n' "$dir/pnpm-install-contract.json" - return 0 - fi - - dir="$(dirname "$dir")" - done - - return 1 -} - pnpm_contract_section_json() { local node_bin="$1" local contract_file="$2" @@ -128,424 +258,6 @@ compute_pnpm_contract_section_hash() { pnpm_contract_section_json "$node_bin" "$contract_file" "$section" | compute_hash } -pnpm_contract_supports_dependency_materialization_profile() { - local node_bin="$1" - local contract_file="$2" - - "$node_bin" - "$contract_file" <<'EOF' -const fs = require('node:fs') - -const [contractFile] = process.argv.slice(2) -const contract = JSON.parse(fs.readFileSync(contractFile, 'utf8')) - -process.exit( - contract.dependencyMaterializationProfile?.schema === 'dependency-materialization-profile/v0' - ? 0 - : 1, -) -EOF -} - -emit_dependency_materialization_profile() { - local node_bin="$1" - local contract_file="$2" - local store_trait="$3" - local output_file="${4:-}" - - "$node_bin" - "$contract_file" "$store_trait" "$output_file" <<'EOF' -const crypto = require('node:crypto') -const fs = require('node:fs') -const path = require('node:path') - -const [contractFile, storeTrait, outputFile] = process.argv.slice(2) -const contract = JSON.parse(fs.readFileSync(contractFile, 'utf8')) -const evidenceContractPath = path.isAbsolute(contractFile) - ? path.relative(fs.realpathSync(process.cwd()), fs.realpathSync(contractFile)) - : contractFile - -const stableJson = (value) => { - if (Array.isArray(value)) { - return value.map(stableJson) - } - - if (value === null || typeof value !== 'object') { - return value - } - - return Object.fromEntries( - Object.entries(value) - .sort(([left], [right]) => left.localeCompare(right)) - .map(([key, nested]) => [key, stableJson(nested)]), - ) -} - -const digest = (value) => - crypto - .createHash('sha256') - .update(`${JSON.stringify(stableJson(value))}\n`) - .digest('hex') - -const profileContract = contract.dependencyMaterializationProfile -if (profileContract?.schema !== 'dependency-materialization-profile/v0') { - console.error(`[pnpm] ${contractFile} has no dependencyMaterializationProfile schema`) - process.exit(1) -} - -const trait = profileContract.supportedTraits?.[storeTrait] -if (trait === undefined) { - console.error(`[pnpm] unsupported dependency materialization store trait '${storeTrait}'`) - process.exit(1) -} - -const inputSections = Object.fromEntries( - profileContract.identityInputs.map((section) => { - if (!Object.prototype.hasOwnProperty.call(contract, section)) { - console.error(`[pnpm] ${contractFile} has no identity section '${section}'`) - process.exit(1) - } - return [section, contract[section]] - }), -) - -const sectionDigests = Object.fromEntries( - Object.entries(inputSections).map(([section, value]) => [section, digest(value)]), -) -const topologyDigest = digest({ - packageManager: inputSections.packageManager, - workspaceManifestContract: inputSections.workspaceManifestContract, -}) -const policyDigest = digest({ - gvsLinkContract: inputSections.gvsLinkContract, - installPolicy: inputSections.installPolicy, -}) -const storeDigest = digest({ - storeContract: inputSections.storeContract, - storeTrait, - trait, -}) - -const profile = { - schema: 'dependency-materialization-profile/v0', - profileId: `pnpm:${topologyDigest}:${policyDigest}:${storeDigest}:${storeTrait}`, - store: { - trait: storeTrait, - contract: inputSections.storeContract, - }, - authorities: { - gc: trait.gcAuthority, - repair: trait.repairAuthority, - }, - topology: { - digest: topologyDigest, - workspaceManifestContractDigest: sectionDigests.workspaceManifestContract, - }, - policy: { - digest: policyDigest, - gvsLinkContractDigest: sectionDigests.gvsLinkContract, - installPolicyDigest: sectionDigests.installPolicy, - nativeBuildPolicyInputs: profileContract.nativeBuildPolicyInputs, - }, - evidence: { - contractPath: evidenceContractPath, - sectionDigests, - }, -} - -const rendered = `${JSON.stringify(profile, null, 2)}\n` -if (outputFile) { - fs.writeFileSync(outputFile, rendered) -} else { - process.stdout.write(rendered) -} -EOF -} - -write_dependency_materialization_registry() { - local node_bin="$1" - local profile_file="$2" - local project_dir="$3" - local store_dir="$4" - local output_file="$5" - local shared_registry_file="${6:-}" - - "$node_bin" - "$profile_file" "$project_dir" "$store_dir" "$output_file" "$shared_registry_file" <<'EOF' -const crypto = require('node:crypto') -const fs = require('node:fs') -const path = require('node:path') - -const [profileFile, projectDir, storeDir, outputFile, sharedRegistryFile] = process.argv.slice(2) -const profile = JSON.parse(fs.readFileSync(profileFile, 'utf8')) -const storeLayoutVersion = 'v11' -const filesPath = path.join(storeDir, storeLayoutVersion, 'files') - -const realFilesPath = (() => { - try { - return fs.realpathSync(filesPath) - } catch { - return filesPath - } -})() - -const poolId = crypto - .createHash('sha256') - .update(`${realFilesPath}\n`) - .digest('hex') -const rootId = crypto - .createHash('sha256') - .update(`${profile.profileId}\n${projectDir}\n${storeDir}\n`) - .digest('hex') - -const singletonRegistry = { - schema: 'dependency-materialization-registry/v0', - profiles: [ - { - id: rootId, - profileId: profile.profileId, - project: projectDir, - store: storeDir, - filesPoolId: poolId, - }, - ], - pools: [ - { - id: poolId, - filesPath, - }, - ], -} - -const readRegistry = (file) => { - if (!file || !fs.existsSync(file)) { - return { schema: 'dependency-materialization-registry/v0', profiles: [], pools: [] } - } - - const registry = JSON.parse(fs.readFileSync(file, 'utf8')) - return { - schema: 'dependency-materialization-registry/v0', - profiles: Array.isArray(registry.profiles) ? registry.profiles : [], - pools: Array.isArray(registry.pools) ? registry.pools : [], - } -} - -const upsertBy = (rows, row, key) => [ - ...rows.filter((candidate) => candidate[key] !== row[key]), - row, -].sort((left, right) => left[key].localeCompare(right[key])) - -const merged = readRegistry(sharedRegistryFile) -const nextProfile = singletonRegistry.profiles[0] -const withoutSameRoot = merged.profiles.filter( - (candidate) => candidate.project !== nextProfile.project || candidate.store !== nextProfile.store, -) -merged.profiles = upsertBy(withoutSameRoot, nextProfile, 'id') -merged.pools = upsertBy(merged.pools, singletonRegistry.pools[0], 'id') - -const rendered = `${JSON.stringify(merged, null, 2)}\n` -if (sharedRegistryFile) { - fs.mkdirSync(path.dirname(sharedRegistryFile), { recursive: true }) - const tmpFile = `${sharedRegistryFile}.${process.pid}.tmp` - fs.writeFileSync(tmpFile, rendered) - fs.renameSync(tmpFile, sharedRegistryFile) -} -fs.writeFileSync(outputFile, rendered) -EOF -} - -dependency_materialization_shared_registry_file() { - local node_bin="$1" - local store_dir="$2" - - "$node_bin" - "$store_dir" <<'EOF' -const fs = require('node:fs') -const path = require('node:path') - -const [storeDir] = process.argv.slice(2) -const storeLayoutVersion = 'v11' -const filesPath = path.join(storeDir, storeLayoutVersion, 'files') -const realFilesPath = (() => { - try { - return fs.realpathSync(filesPath) - } catch { - return filesPath - } -})() - -process.stdout.write(path.join( - path.dirname(realFilesPath), - `.effect-utils-dependency-materialization-registry-${storeLayoutVersion}.json`, -)) -EOF -} - -dependency_materialization_profile_id() { - local node_bin="$1" - local profile_file="$2" - - "$node_bin" - "$profile_file" <<'EOF' -const fs = require('node:fs') - -const [profileFile] = process.argv.slice(2) -const profile = JSON.parse(fs.readFileSync(profileFile, 'utf8')) -process.stdout.write(profile.profileId) -EOF -} - -dependency_materialization_profile_files_pool_id() { - local node_bin="$1" - local registry_file="$2" - local profile_id="$3" - - "$node_bin" - "$registry_file" "$profile_id" <<'EOF' -const fs = require('node:fs') - -const [registryFile, profileId] = process.argv.slice(2) -const registry = JSON.parse(fs.readFileSync(registryFile, 'utf8')) -const profiles = Array.isArray(registry.profiles) ? registry.profiles : [] -const profile = profiles.find((row) => row.id === profileId || row.profileId === profileId) - -if (profile === undefined || typeof profile.filesPoolId !== 'string') { - process.exit(1) -} - -process.stdout.write(profile.filesPoolId) -EOF -} - -dependency_materialization_profile_store_dir() { - local node_bin="$1" - local registry_file="$2" - local profile_id="$3" - - "$node_bin" - "$registry_file" "$profile_id" <<'EOF' -const fs = require('node:fs') - -const [registryFile, profileId] = process.argv.slice(2) -const registry = JSON.parse(fs.readFileSync(registryFile, 'utf8')) -const profiles = Array.isArray(registry.profiles) ? registry.profiles : [] -const profile = profiles.find((row) => row.id === profileId || row.profileId === profileId) - -if (profile === undefined || typeof profile.store !== 'string') { - process.exit(1) -} - -process.stdout.write(profile.store) -EOF -} - -dependency_materialization_repair_roots() { - local node_bin="$1" - local registry_file="$2" - local files_pool_id="$3" - - "$node_bin" - "$registry_file" "$files_pool_id" <<'EOF' -const fs = require('node:fs') - -const [registryFile, filesPoolId] = process.argv.slice(2) -const registry = JSON.parse(fs.readFileSync(registryFile, 'utf8')) -const profiles = Array.isArray(registry.profiles) ? registry.profiles : [] - -for (const profile of profiles - .filter((row) => row.filesPoolId === filesPoolId) - .sort((left, right) => left.id.localeCompare(right.id))) { - process.stdout.write(`${profile.project}\t${profile.store}\n`) -} -EOF -} - -dependency_materialization_store_doctor() { - local node_bin="$1" - local registry_file="$2" - local profile_id="$3" - - "$node_bin" - "$registry_file" "$profile_id" <<'EOF' -const fs = require('node:fs') -const path = require('node:path') - -const [registryFile, profileId] = process.argv.slice(2) -const registry = JSON.parse(fs.readFileSync(registryFile, 'utf8')) - -const classifyPool = (pool) => { - if (pool.filesKind !== undefined) return pool.filesKind - if (typeof pool.filesPath !== 'string') return 'missing' - - try { - const stat = fs.lstatSync(pool.filesPath) - if (stat.isSymbolicLink()) { - const target = fs.realpathSync(pool.filesPath) - const localRoot = path.dirname(path.dirname(pool.filesPath)) - return target.startsWith(`${localRoot}${path.sep}`) - ? 'profile-local-symlink' - : 'shared-symlink' - } - if (stat.isDirectory()) return 'directory' - return 'invalid' - } catch { - return 'missing' - } -} - -const profiles = Array.isArray(registry.profiles) ? registry.profiles : [] -const pools = Array.isArray(registry.pools) ? registry.pools : [] -const profile = profiles.find((row) => row.id === profileId || row.profileId === profileId) - -if (profile === undefined) { - console.log(JSON.stringify({ phase: 'doctor', profileId, decision: 'refuse', reason: 'unknown-profile', siblings: [] })) - process.exit(0) -} - -const pool = pools.find((row) => row.id === profile.filesPoolId) -if (pool === undefined) { - console.log(JSON.stringify({ phase: 'doctor', profileId, filesPoolId: profile.filesPoolId, decision: 'refuse', reason: 'unknown-files-pool', siblings: [] })) - process.exit(0) -} - -const filesKind = classifyPool(pool) -const siblings = profiles - .filter((row) => row.filesPoolId === pool.id) - .map((row) => row.id) - .sort() - -if ((filesKind === 'directory' || filesKind === 'profile-local-symlink') && siblings.length === 1) { - console.log(JSON.stringify({ phase: 'doctor', profileId, filesPoolId: pool.id, decision: 'allow-profile-local-prune', reason: 'profile-local-files-pool', siblings, filesKind })) - process.exit(0) -} - -console.log(JSON.stringify({ - phase: 'doctor', - profileId, - filesPoolId: pool.id, - decision: 'refuse-raw-prune', - reason: filesKind === 'shared-symlink' ? 'shared-files-pool' : 'invalid-files-pool', - siblings, - filesKind, -})) -EOF -} - -dependency_materialization_repair_plan() { - local node_bin="$1" - local registry_file="$2" - local files_pool_id="$3" - - "$node_bin" - "$registry_file" "$files_pool_id" <<'EOF' -const fs = require('node:fs') - -const [registryFile, filesPoolId] = process.argv.slice(2) -const registry = JSON.parse(fs.readFileSync(registryFile, 'utf8')) -const profiles = Array.isArray(registry.profiles) ? registry.profiles : [] -const roots = profiles - .filter((profile) => profile.filesPoolId === filesPoolId) - .map((profile) => ({ profile: profile.id, project: profile.project, store: profile.store })) - .sort((left, right) => left.profile.localeCompare(right.profile)) - -if (roots.length === 0) { - console.log(JSON.stringify({ phase: 'repair-plan', filesPoolId, decision: 'refuse', reason: 'no-registered-roots', roots })) -} else { - console.log(JSON.stringify({ phase: 'repair-plan', filesPoolId, decision: 'repair-all-roots', reason: 'registered-roots', roots })) -} -EOF -} - classify_pnpm_contract_change() { local node_bin="$1" local previous_contract="$2" @@ -590,7 +302,7 @@ const currentContract = JSON.parse(fs.readFileSync(currentContractFile, 'utf8')) for (const [section, reason] of [ ['packageManager', 'toolchain'], - ['gvsLinkContract', 'gvs-link'], + ['dependencyGraphContract', 'dependency_graph'], ['installPolicy', 'policy'], ['storeContract', 'store'], ['workspaceManifestContract', 'manifest_config'], diff --git a/nix/devenv-modules/tasks/shared/pnpm.nix b/nix/devenv-modules/tasks/shared/pnpm.nix index 31c9468982..e67863e17f 100644 --- a/nix/devenv-modules/tasks/shared/pnpm.nix +++ b/nix/devenv-modules/tasks/shared/pnpm.nix @@ -19,6 +19,10 @@ frozenInCi ? true, installFlags ? [ ], preInstall ? "", + # Root-owned immutable projections that must be part of the authoritative + # materialized topology. Runs after pnpm and its base health oracle, before + # the projection digest is committed. + postInstallProjection ? "", installAfter ? [ ], updateAfter ? [ ], dedupeAfter ? [ ], @@ -60,7 +64,7 @@ let "${config.devenv.root}/.devenv/pnpm-home" else "${config.devenv.root}/.devenv/pnpm-home/${workspaceCacheName}"; - defaultPnpmStoreDir = + jobLocalPnpmStoreDir = if workspaceRoot == "." then "${config.devenv.root}/.devenv/pnpm-store-pure-v1" else @@ -78,13 +82,13 @@ let if taskSuffix == null then "${taskNamePrefix}:clean" else "${taskNamePrefix}:clean:${taskSuffix}"; doctorTaskName = if taskSuffix == null then "${taskNamePrefix}:doctor" else "${taskNamePrefix}:doctor:${taskSuffix}"; - repairPlanTaskName = - if taskSuffix == null then - "${taskNamePrefix}:repair-plan" - else - "${taskNamePrefix}:repair-plan:${taskSuffix}"; repairTaskName = if taskSuffix == null then "${taskNamePrefix}:repair" else "${taskNamePrefix}:repair:${taskSuffix}"; + migrateLegacyStoreTaskName = + if taskSuffix == null then + "${taskNamePrefix}:store:migrate-legacy" + else + "${taskNamePrefix}:store:migrate-legacy:${taskSuffix}"; resetLockFilesTaskName = if taskSuffix == null then "${taskNamePrefix}:reset-lock-files" @@ -113,11 +117,15 @@ let flock = "${pkgs.flock}/bin/flock"; installFlagsString = lib.escapeShellArgs installFlags; - pureInstallFlags = [ - (if frozenInCi then "--frozen-lockfile" else "--no-frozen-lockfile") - ] - ++ pnpmInstallPolicy.liveInstallPolicyFlags; - pureInstallFlagsString = lib.concatStringsSep " " pureInstallFlags; + liveRealizationPolicyFlags = installFlags ++ pnpmInstallPolicy.liveInstallPolicyFlags; + liveRealizationPolicyFlagsString = lib.escapeShellArgs liveRealizationPolicyFlags; + pureInstallFlags = + installFlags + ++ [ + (if frozenInCi then "--frozen-lockfile" else "--no-frozen-lockfile") + ] + ++ pnpmInstallPolicy.liveInstallPolicyFlags; + pureInstallFlagsString = lib.escapeShellArgs pureInstallFlags; packageNameToPath = builtins.listToAttrs ( builtins.filter (x: x != null) ( @@ -171,8 +179,8 @@ let source ${lib.escapeShellArg pnpmTaskHelpersScript} ''; ensureLocalPnpmHomeFn = '' - # Keep pnpm's hot GVS projection workspace-local by default so local tasks - # match CI and don't inherit stale global link state from unrelated repos. + # Keep root-owned package-manager state workspace-local. The complete pnpm + # package store is shared, including pnpm's concurrency-safe derived index. if [ ${lib.escapeShellArg workspaceRoot} = "." ]; then if [ -z "''${PNPM_HOME:-}" ]; then export PNPM_HOME=${lib.escapeShellArg defaultPnpmHome} @@ -186,61 +194,45 @@ let esac fi ''; - ensureLocalPnpmStoreDirFn = '' - _pnpm_store_dir="''${npm_config_store_dir:-''${PNPM_CONFIG_STORE_DIR:-''${PNPM_STORE_DIR:-}}}" - if [ ${lib.escapeShellArg workspaceRoot} != "." ] && [ -n "$_pnpm_store_dir" ]; then - case "$_pnpm_store_dir" in - */${workspaceCacheName}) ;; - *) _pnpm_store_dir="$_pnpm_store_dir/${workspaceCacheName}" ;; - esac - elif [ -n "$_pnpm_store_dir" ]; then - : - else - _pnpm_store_dir=${lib.escapeShellArg defaultPnpmStoreDir} - fi - export PNPM_STORE_DIR="$_pnpm_store_dir" - export PNPM_CONFIG_STORE_DIR="$_pnpm_store_dir" - export npm_config_store_dir="$_pnpm_store_dir" - unset _pnpm_store_dir + configurePnpmStorageFn = '' + configure_pnpm_storage \ + ${lib.escapeShellArg "${pkgs.nodejs}/bin/node"} \ + ${lib.escapeShellArg workspaceRootAbs} \ + ${lib.escapeShellArg jobLocalPnpmStoreDir} \ + ${lib.boolToString pkgs.stdenv.hostPlatform.isLinux} ''; - ensureSharedPnpmFilesStoreFn = '' - ensure_shared_pnpm_files_store() { - if [ -n "''${CI:-}" ]; then - return 0 - fi - if [ -z "''${npm_config_store_dir:-}" ]; then - echo "[pnpm] npm_config_store_dir is empty; cannot prepare split store" >&2 - exit 1 - fi - - local store_version_dir - local files_path - local shared_files_path - store_version_dir="''${npm_config_store_dir}/v11" - files_path="$store_version_dir/files" - shared_files_path="''${PNPM_SHARED_FILES_DIR:-$HOME/.local/share/pnpm/shared-files}/v11" - - mkdir -p "$store_version_dir" "$shared_files_path" + managedPnpmMutationPrologue = '' + ${loadPnpmTaskHelpersFn} + ${ensureLocalPnpmHomeFn} + ${configurePnpmStorageFn} + mkdir -p ${lib.escapeShellArg cacheRoot} + + lockfile=${lib.escapeShellArg "${cacheRoot}/pnpm-install.lock"} + exec 200>"$lockfile" + if ! ${flock} -w 600 200; then + echo "[pnpm] Materialization-root mutation lock timeout after 600s: $lockfile" >&2 + echo "[pnpm] Another managed pnpm mutation may be stuck" >&2 + exit 1 + fi - if [ -L "$files_path" ]; then - if [ "$(readlink "$files_path")" != "$shared_files_path" ]; then - echo "[pnpm] $files_path points at $(readlink "$files_path"), expected $shared_files_path" >&2 - exit 1 - fi - return 0 - fi + pnpm_home_lockfile="''${PNPM_HOME:-${cacheRoot}}/.effect-utils-pnpm-install.lock" + mkdir -p "$(dirname "$pnpm_home_lockfile")" + exec 201>"$pnpm_home_lockfile" + if ! ${flock} -w 600 201; then + echo "[pnpm] PNPM_HOME mutation lock timeout after 600s: $pnpm_home_lockfile" >&2 + echo "[pnpm] Another managed pnpm mutation sharing this PNPM_HOME may be stuck" >&2 + exit 1 + fi - if [ -e "$files_path" ]; then - if [ -d "$files_path" ] && [ -z "$(find "$files_path" -mindepth 1 -maxdepth 1 -print -quit)" ]; then - rmdir "$files_path" - else - echo "[pnpm] $files_path is a non-empty local files store; leaving it for the coordinated migration runbook" >&2 - return 0 - fi - fi + # Installs sharing a Store Cache take compatible shared admission leases. + # Host-owned maintenance takes the exclusive counterpart, so pruning can + # never race pnpm while independent Materialization Roots stay concurrent. + acquire_pnpm_store_cache_lease ${lib.escapeShellArg flock} shared "$npm_config_store_dir" 600 - ln -s "$shared_files_path" "$files_path" - } + assert_pnpm_storage_capacity \ + ${lib.escapeShellArg "${pkgs.nodejs}/bin/node"} \ + "$npm_config_store_dir" \ + ${lib.escapeShellArg workspaceRootAbs} ''; computeWorkspaceStateHash = '' @@ -269,17 +261,16 @@ let computeInstallStateHashFn = '' compute_install_state_hash() { local workspace_state_hash - local gvs_links_dir - workspace_state_hash="$(compute_workspace_state_hash)" - gvs_links_dir="$(resolve_gvs_links_dir)" { printf '%s\n' ${lib.escapeShellArg pkgs.pnpm.version} printf '%s\n' "$workspace_state_hash" - printf '%s\n' "''${gvs_links_dir:-}" + printf '%s\n' "$npm_config_store_dir" + printf '%s\n' "$PNPM_PACKAGE_IMPORT_METHOD" printf '%s\n' ${lib.escapeShellArg (builtins.toJSON installFlags)} printf '%s\n' ${lib.escapeShellArg preInstall} + printf '%s\n' ${lib.escapeShellArg postInstallProjection} } | compute_hash } ''; @@ -290,7 +281,6 @@ let # ordered line stream that the previous bash implementation produced. NODE_MODULES_HELPER_MODE="projection-hash" \ PNPM_ROOT_MODULES_YAML="node_modules/.modules.yaml" \ - PNPM_GVS_LINKS_DIR="$(resolve_gvs_links_dir)" \ NODE_MODULES_DIRS="$(printf '%s\n' node_modules ${nodeModulesPaths})" \ ${pkgs.nodejs}/bin/node ${lib.escapeShellArg nodeModulesProjectionScript} } @@ -314,6 +304,11 @@ let --strict-store-pkg-content-check=false | --no-strict-store-pkg-content-check | \ --config.strict-store-pkg-content-check=false | --config.strict-store-pkg-content-check | \ --config.manage-package-manager-versions=true | --config.manage-package-manager-versions | \ + --config.enable-global-virtual-store=true | --enable-global-virtual-store | \ + --config.global-virtual-store-dir=* | --config.global-virtual-store-dir | \ + --global-virtual-store-dir=* | --global-virtual-store-dir | \ + --config.virtual-store-dir=* | --config.virtual-store-dir | \ + --virtual-store-dir=* | --virtual-store-dir | \ --pm-on-fail=* | --pm-on-fail | --config.pm-on-fail=* | --config.pm-on-fail | \ --config.package-import-method=* | --config.package-import-method | --package-import-method=* | --package-import-method | \ --config.store-dir=* | --config.store-dir | --store-dir=* | --store-dir) @@ -327,7 +322,13 @@ let run_pnpm_install() { local install_args reject_impure_pnpm_install_args "$@" ${installFlagsString} - install_args=(install "$@" ${installFlagsString} ${pureInstallFlagsString} "--config.store-dir=$npm_config_store_dir") + install_args=( + install + "$@" + ${pureInstallFlagsString} + "--config.package-import-method=$PNPM_PACKAGE_IMPORT_METHOD" + "--config.store-dir=$npm_config_store_dir" + ) ${lib.optionalString pkgs.stdenv.hostPlatform.isDarwin '' if [ -n "''${CI:-}" ]; then @@ -455,7 +456,10 @@ let fi set +e - ${lib.escapeShellArg "${effectivePnpmLockMutatorPkg}/bin/pnpm"} install --fix-lockfile --config.confirmModulesPurge=false --pm-on-fail=ignore --config.store-dir="$npm_config_store_dir" + ${lib.escapeShellArg "${effectivePnpmLockMutatorPkg}/bin/pnpm"} install --fix-lockfile \ + ${liveRealizationPolicyFlagsString} \ + --config.package-import-method="$PNPM_PACKAGE_IMPORT_METHOD" \ + --config.store-dir="$npm_config_store_dir" status=$? set -e @@ -499,46 +503,13 @@ let exec = trace.exec installTaskName '' set -euo pipefail cd ${lib.escapeShellArg workspaceRootAbs} - ${loadPnpmTaskHelpersFn} - ${ensureLocalPnpmHomeFn} - ${ensureLocalPnpmStoreDirFn} - ${ensureSharedPnpmFilesStoreFn} - ensure_shared_pnpm_files_store - mkdir -p "${cacheRoot}" + ${managedPnpmMutationPrologue} # This cache tracks the effective install state, not just workspace - # manifests. The fingerprint also includes the active GVS projection - # root because pnpm 11 bakes absolute paths into `links/`. + # manifests. The virtual dependency graph itself is root-local. hash_file="${cacheRoot}/install-state.hash" projection_hash_file="${cacheRoot}/projection-state.hash" contract_state_file="${cacheRoot}/pnpm-install-contract.json" - dependency_profile_file="${cacheRoot}/dependency-materialization-profile.json" - dependency_registry_file="${cacheRoot}/dependency-materialization-registry.json" - - lockfile="${cacheRoot}/pnpm-install.lock" - exec 200>"$lockfile" - if ! ${flock} -w 600 200; then - echo "[pnpm] Install lock timeout after 600s: $lockfile" >&2 - echo "[pnpm] Another pnpm install may be stuck; try: devenv tasks run pnpm:clean && devenv tasks run pnpm:install" >&2 - exit 1 - fi - - pnpm_home_lockfile="''${PNPM_HOME:-${cacheRoot}}/.effect-utils-pnpm-install.lock" - mkdir -p "$(dirname "$pnpm_home_lockfile")" - exec 201>"$pnpm_home_lockfile" - if ! ${flock} -w 600 201; then - echo "[pnpm] PNPM_HOME lock timeout after 600s: $pnpm_home_lockfile" >&2 - echo "[pnpm] Another pnpm install sharing this PNPM_HOME may be stuck" >&2 - exit 1 - fi - - pnpm_store_lockfile="''${npm_config_store_dir:-${cacheRoot}}/.effect-utils-pnpm-store.lock" - mkdir -p "$(dirname "$pnpm_store_lockfile")" - exec 202>"$pnpm_store_lockfile" - if ! ${flock} -w 600 202; then - echo "[pnpm] store-dir lock timeout after 600s: $pnpm_store_lockfile" >&2 - echo "[pnpm] Another pnpm install sharing this store-dir may be stuck" >&2 - exit 1 - fi + storage_state_file="${cacheRoot}/pnpm-storage-state" ${computeWorkspaceStateHash} ${computeInstallStateHashFn} @@ -546,64 +517,22 @@ let ${preInstall} ${runPnpmInstallFn} - # pnpm 11 GVS: hash-based link invalidation. pnpm reuses existing GVS - # entries without re-resolving packageExtensions, so stale entries break - # TypeScript resolution. Only clear links/ when config changes. - # Content-addressable store (files/) is unaffected. - # See: pnpm/pnpm#9739 - _pnpm_install_contract_file="$(resolve_pnpm_install_contract_file "$PWD" || true)" - if [ -n "''${_pnpm_install_contract_file:-}" ]; then - _gvs_hash="$(compute_pnpm_contract_section_hash ${pkgs.nodejs}/bin/node "$_pnpm_install_contract_file" gvsLinkContract)" - else + _pnpm_install_contract_file="$PWD/pnpm-install-contract.json" + if [ ! -f "$_pnpm_install_contract_file" ]; then + _pnpm_install_contract_file="" ${lib.optionalString (workspaceRoot == ".") '' echo "[pnpm] Missing generated pnpm-install-contract.json at repo root" >&2 echo "[pnpm] Run: devenv tasks run genie:run" >&2 exit 1 ''} - # Non-root downstream workspaces may not carry the generated contract - # yet. Keep the fallback deliberately coarse and structured: no YAML - # parsing, no partial pnpm-owned layout inference. - _gvs_hash="$(printf '%s\n' ${lib.escapeShellArg pkgs.pnpm.version} | compute_hash)" + : # A nested downstream root may not emit profile evidence yet. fi - _gvs_hash_file="" - _gvs_links_dir="$(resolve_gvs_links_dir)" - _purged_node_modules=false _force_install=false - if [ -n "''${_gvs_links_dir:-}" ]; then - _gvs_hash_file="$(dirname "$_gvs_links_dir")/.effect-utils-gvs-links.hash" - mkdir -p "$(dirname "$_gvs_links_dir")" - if [ ! -f "$_gvs_hash_file" ] || [ "$(cat "$_gvs_hash_file")" != "$_gvs_hash" ]; then - echo "[pnpm] GVS config changed, forcing current workspace relink" - purge_node_modules node_modules ${nodeModulesPaths} - # A workspace relink only rewrites node_modules. If the broken - # package projection is already cached under v11/links, pnpm can - # reuse that incomplete directory even for `pnpm install --force`. - # Dropping links/ keeps the content-addressed files/ store intact - # while forcing GVS to materialize fresh package link projections. - # See https://github.com/pnpm/pnpm/issues/11385. - # TODO(pnpm#11385): remove this links/ purge once forced installs - # rebuild incomplete GVS link projections. - rm -rf "$_gvs_links_dir" - _purged_node_modules=true - _force_install=true - fi - fi - - if [ "$_purged_node_modules" != true ] && ! check_node_modules_links_healthy ${pkgs.nodejs}/bin/node ${lib.escapeShellArg nodeModulesProjectionScript} ${healthCheckNodeModulesPaths}; then + if ! check_node_modules_links_healthy ${pkgs.nodejs}/bin/node ${lib.escapeShellArg nodeModulesProjectionScript} ${healthCheckNodeModulesPaths}; then echo "[pnpm] node_modules projection is stale, purging install state" purge_node_modules node_modules ${nodeModulesPaths} - if [ -n "''${_gvs_links_dir:-}" ]; then - # The health check can fail while package symlinks and package.json - # still exist, e.g. an exported runtime file is missing inside a GVS - # link projection. Deleting node_modules alone would just reconnect - # the workspace to the same incomplete v11/links package directory. - # See https://github.com/pnpm/pnpm/issues/11385. - # TODO(pnpm#11385): remove this links/ purge once forced installs - # rebuild incomplete GVS link projections. - rm -rf "$_gvs_links_dir" - fi _force_install=true fi @@ -618,32 +547,19 @@ let exit 1 fi - # Persist GVS hash after successful install - if [ -n "''${_gvs_hash_file:-}" ]; then - echo "$_gvs_hash" > "$_gvs_hash_file" + ${postInstallProjection} + + if ! check_node_modules_links_healthy ${pkgs.nodejs}/bin/node ${lib.escapeShellArg nodeModulesProjectionScript} ${healthCheckNodeModulesPaths}; then + echo "[pnpm] node_modules projection is unhealthy after the root-owned post-install projection" >&2 + exit 1 fi + if [ -n "''${_pnpm_install_contract_file:-}" ]; then rm -f "$contract_state_file" cp "$_pnpm_install_contract_file" "$contract_state_file" chmod u+w "$contract_state_file" 2>/dev/null || true - if pnpm_contract_supports_dependency_materialization_profile ${pkgs.nodejs}/bin/node "$_pnpm_install_contract_file"; then - if [ -n "''${CI:-}" ]; then - _dependency_materialization_trait="ciJobLocal" - elif [ -L "$npm_config_store_dir/v11/files" ]; then - _dependency_materialization_trait="darwinSplitCas" - else - _dependency_materialization_trait="isolated" - fi - emit_dependency_materialization_profile ${pkgs.nodejs}/bin/node "$_pnpm_install_contract_file" "$_dependency_materialization_trait" "$dependency_profile_file" - _dependency_shared_registry_file="$(dependency_materialization_shared_registry_file ${pkgs.nodejs}/bin/node "$npm_config_store_dir")" - mkdir -p "$(dirname "$_dependency_shared_registry_file")" - exec 203>"$_dependency_shared_registry_file.lock" - if ! ${flock} -w 600 203; then - echo "[pnpm] dependency materialization registry lock timeout after 600s: $_dependency_shared_registry_file.lock" >&2 - exit 1 - fi - write_dependency_materialization_registry ${pkgs.nodejs}/bin/node "$dependency_profile_file" "$PWD" "$npm_config_store_dir" "$dependency_registry_file" "$_dependency_shared_registry_file" - fi + else + rm -f "$contract_state_file" fi cache_value="$(compute_install_state_hash)" @@ -651,46 +567,47 @@ let cache_value="$(compute_projection_state_hash)" ${cache.writeCacheFile ''"$projection_hash_file"''} + + cache_value="$(printf '%s\n%s\n' "$npm_config_store_dir" "$PNPM_PACKAGE_IMPORT_METHOD")" + ${cache.writeCacheFile ''"$storage_state_file"''} ''; status = trace.status installTaskName "hash" '' set -euo pipefail cd ${lib.escapeShellArg workspaceRootAbs} ${loadPnpmTaskHelpersFn} ${ensureLocalPnpmHomeFn} - ${ensureLocalPnpmStoreDirFn} - ${ensureSharedPnpmFilesStoreFn} - ensure_shared_pnpm_files_store + ${configurePnpmStorageFn} hash_file="${cacheRoot}/install-state.hash" projection_hash_file="${cacheRoot}/projection-state.hash" contract_state_file="${cacheRoot}/pnpm-install-contract.json" - dependency_profile_file="${cacheRoot}/dependency-materialization-profile.json" - dependency_registry_file="${cacheRoot}/dependency-materialization-registry.json" + storage_state_file="${cacheRoot}/pnpm-storage-state" - _pnpm_install_contract_file="$(resolve_pnpm_install_contract_file "$PWD" || true)" - if [ -z "''${_pnpm_install_contract_file:-}" ]; then + _pnpm_install_contract_file="$PWD/pnpm-install-contract.json" + if [ ! -f "$_pnpm_install_contract_file" ]; then + _pnpm_install_contract_file="" ${lib.optionalString (workspaceRoot == ".") '' echo "[pnpm] Missing generated pnpm-install-contract.json at repo root" >&2 echo "[pnpm] Run: devenv tasks run genie:run" >&2 + emit_pnpm_install_miss_span ${lib.escapeShellArg installTaskName} "contract_missing" + exit 1 ''} - emit_pnpm_install_miss_span ${lib.escapeShellArg installTaskName} "contract_missing" - exit 1 fi - if [ ! -d node_modules ] || [ ! -f pnpm-lock.yaml ] || [ ! -f "$hash_file" ] || [ ! -f "$projection_hash_file" ] || [ ! -f node_modules/.modules.yaml ]; then + if [ ! -d node_modules ] || [ ! -f pnpm-lock.yaml ] || [ ! -f "$hash_file" ] || [ ! -f "$projection_hash_file" ] || [ ! -f "$storage_state_file" ] || [ ! -f node_modules/.modules.yaml ]; then emit_pnpm_install_miss_span ${lib.escapeShellArg installTaskName} "bootstrap" exit 1 fi - if pnpm_contract_supports_dependency_materialization_profile ${pkgs.nodejs}/bin/node "$_pnpm_install_contract_file" && { [ ! -f "$dependency_profile_file" ] || [ ! -f "$dependency_registry_file" ]; }; then - emit_pnpm_install_miss_span ${lib.escapeShellArg installTaskName} "bootstrap" + current_storage_state="$(printf '%s\n%s\n' "$npm_config_store_dir" "$PNPM_PACKAGE_IMPORT_METHOD")" + if [ "$current_storage_state" != "$(cat "$storage_state_file")" ]; then + emit_pnpm_install_miss_span ${lib.escapeShellArg installTaskName} "storage_policy" exit 1 fi if [ "''${DEVENV_SETUP_OUTER_CACHE_HIT:-0}" = "1" ]; then - # Keep shell entry fast by reusing the cached install-state proof and - # only re-validating the realized projection structure here. The full - # semantic health check still runs in the exec path before install can - # be treated as clean again. + # The stored projection digest was written only after the full health + # oracle passed. Compare the complete realization evidence instead of + # repeating module resolution for every cached downstream task. ${computeProjectionStateHashFn} current_projection_hash="$(compute_projection_state_hash)" stored_projection_hash="$(cat "$projection_hash_file")" @@ -709,7 +626,7 @@ let stored_hash="$(cat "$hash_file")" stored_projection_hash="$(cat "$projection_hash_file")" if [ "$current_hash" != "$stored_hash" ]; then - if [ -f "$contract_state_file" ]; then + if [ -f "$contract_state_file" ] && [ -n "''${_pnpm_install_contract_file:-}" ]; then _miss_reason="$(classify_pnpm_contract_change ${pkgs.nodejs}/bin/node "$contract_state_file" "$_pnpm_install_contract_file" || printf '%s\n' unknown)" else _miss_reason="unknown" @@ -732,11 +649,7 @@ let exec = trace.exec updateTaskName '' set -euo pipefail cd ${lib.escapeShellArg workspaceRootAbs} - ${loadPnpmTaskHelpersFn} - ${ensureLocalPnpmHomeFn} - ${ensureLocalPnpmStoreDirFn} - ${ensureSharedPnpmFilesStoreFn} - ensure_shared_pnpm_files_store + ${managedPnpmMutationPrologue} ${runPnpmLockMutatorFn} ${lib.optionalString (workspaceRoot == ".") '' # Projection can change the catalog that the lockfile must satisfy. @@ -763,12 +676,10 @@ let exec = trace.exec dedupeTaskName '' set -euo pipefail cd ${lib.escapeShellArg workspaceRootAbs} - ${loadPnpmTaskHelpersFn} - ${ensureLocalPnpmHomeFn} - ${ensureLocalPnpmStoreDirFn} - ${ensureSharedPnpmFilesStoreFn} - ensure_shared_pnpm_files_store - pnpm dedupe --config.confirmModulesPurge=false --pm-on-fail=ignore --config.store-dir="$npm_config_store_dir" + ${managedPnpmMutationPrologue} + pnpm dedupe ${liveRealizationPolicyFlagsString} \ + --config.package-import-method="$PNPM_PACKAGE_IMPORT_METHOD" \ + --config.store-dir="$npm_config_store_dir" echo "Lockfile deduped. Re-run genie:check to verify the catalog duplicate gate; bless any upstream-locked residuals via catalogDuplicateExceptions." ''; }; @@ -782,15 +693,9 @@ let cd ${lib.escapeShellArg workspaceRootAbs} ${loadPnpmTaskHelpersFn} ${ensureLocalPnpmHomeFn} - ${ensureLocalPnpmStoreDirFn} - ${ensureSharedPnpmFilesStoreFn} - ensure_shared_pnpm_files_store + ${configurePnpmStorageFn} purge_node_modules node_modules ${nodeModulesPaths} - - # The GVS `links/` directory lives under the shared store-dir. Deleting - # it from one workspace would break node_modules projections in other - # workspaces that point at the same shared store. ''; }; @@ -802,108 +707,51 @@ let cd ${lib.escapeShellArg workspaceRootAbs} ${loadPnpmTaskHelpersFn} - dependency_profile_file="${cacheRoot}/dependency-materialization-profile.json" - dependency_registry_file="${cacheRoot}/dependency-materialization-registry.json" - if [ ! -f "$dependency_profile_file" ] || [ ! -f "$dependency_registry_file" ]; then - echo "[pnpm] Missing dependency materialization evidence; run: ${installTaskName}" >&2 - exit 1 + if [ -d node_modules/.pnpm ] && check_node_modules_links_healthy ${pkgs.nodejs}/bin/node ${lib.escapeShellArg nodeModulesProjectionScript} ${healthCheckNodeModulesPaths}; then + doctor_decision="healthy" + doctor_reason="root-local-graph-healthy" + else + doctor_decision="repair-root" + doctor_reason="root-local-graph-unhealthy" fi - - profile_id="$(dependency_materialization_profile_id ${pkgs.nodejs}/bin/node "$dependency_profile_file")" - dependency_materialization_store_doctor ${pkgs.nodejs}/bin/node "$dependency_registry_file" "$profile_id" + ${pkgs.nodejs}/bin/node - "$PWD" "$doctor_decision" "$doctor_reason" <<'EOF' + const [root, decision, reason] = process.argv.slice(2) + console.log(JSON.stringify({ phase: 'doctor', root, decision, reason })) + EOF ''; }; - "${repairPlanTaskName}" = { + "${repairTaskName}" = { guard = "pnpm"; - description = "Plan repair from existing dependency materialization evidence for the pnpm workspace at ${workspaceRoot}"; - exec = trace.exec repairPlanTaskName '' + description = "Discard and rematerialize the root-local pnpm dependency graph at ${workspaceRoot}"; + exec = trace.exec repairTaskName '' set -euo pipefail cd ${lib.escapeShellArg workspaceRootAbs} ${loadPnpmTaskHelpersFn} - - dependency_profile_file="${cacheRoot}/dependency-materialization-profile.json" - dependency_registry_file="${cacheRoot}/dependency-materialization-registry.json" - if [ ! -f "$dependency_profile_file" ] || [ ! -f "$dependency_registry_file" ]; then - echo "[pnpm] Missing dependency materialization evidence; run: ${installTaskName}" >&2 - exit 1 - fi - - profile_id="$(dependency_materialization_profile_id ${pkgs.nodejs}/bin/node "$dependency_profile_file")" - profile_store_dir="$(dependency_materialization_profile_store_dir ${pkgs.nodejs}/bin/node "$dependency_registry_file" "$profile_id")" - shared_registry_file="$(dependency_materialization_shared_registry_file ${pkgs.nodejs}/bin/node "$profile_store_dir")" - repair_registry_file="$dependency_registry_file" - if [ -f "$shared_registry_file" ]; then - repair_registry_file="$shared_registry_file" - fi - if ! files_pool_id="$(dependency_materialization_profile_files_pool_id ${pkgs.nodejs}/bin/node "$repair_registry_file" "$profile_id")"; then - files_pool_id="$(dependency_materialization_profile_files_pool_id ${pkgs.nodejs}/bin/node "$dependency_registry_file" "$profile_id")" - fi - dependency_materialization_repair_plan ${pkgs.nodejs}/bin/node "$repair_registry_file" "$files_pool_id" + purge_node_modules node_modules ${nodeModulesPaths} + rm -f \ + "${cacheRoot}/install-state.hash" \ + "${cacheRoot}/projection-state.hash" \ + "${cacheRoot}/pnpm-storage-state" + echo "[pnpm] Discarded root-local dependency graph; reinvoking ${installTaskName}" + exec devenv tasks run ${lib.escapeShellArg installTaskName} ''; }; - "${repairTaskName}" = { + "${migrateLegacyStoreTaskName}" = { guard = "pnpm"; - description = "Repair all registered pnpm workspaces sharing this dependency materialization files pool"; - exec = trace.exec repairTaskName '' + description = "Replace the recognized legacy pnpm files bridge with a self-contained Store Cache"; + exec = trace.exec migrateLegacyStoreTaskName '' set -euo pipefail - cd ${lib.escapeShellArg workspaceRootAbs} - ${loadPnpmTaskHelpersFn} - - dependency_profile_file="${cacheRoot}/dependency-materialization-profile.json" - dependency_registry_file="${cacheRoot}/dependency-materialization-registry.json" - if [ ! -f "$dependency_profile_file" ] || [ ! -f "$dependency_registry_file" ]; then - echo "[pnpm] Missing dependency materialization evidence; run: ${installTaskName}" >&2 - exit 1 - fi - - profile_id="$(dependency_materialization_profile_id ${pkgs.nodejs}/bin/node "$dependency_profile_file")" - profile_store_dir="$(dependency_materialization_profile_store_dir ${pkgs.nodejs}/bin/node "$dependency_registry_file" "$profile_id")" - shared_registry_file="$(dependency_materialization_shared_registry_file ${pkgs.nodejs}/bin/node "$profile_store_dir")" - repair_registry_file="$dependency_registry_file" - if [ -f "$shared_registry_file" ]; then - repair_registry_file="$shared_registry_file" - fi - if ! files_pool_id="$(dependency_materialization_profile_files_pool_id ${pkgs.nodejs}/bin/node "$repair_registry_file" "$profile_id")"; then - files_pool_id="$(dependency_materialization_profile_files_pool_id ${pkgs.nodejs}/bin/node "$dependency_registry_file" "$profile_id")" - fi - dependency_materialization_repair_plan ${pkgs.nodejs}/bin/node "$repair_registry_file" "$files_pool_id" - - repaired_roots=0 - while IFS=$'\t' read -r repair_project_dir repair_store_dir; do - if [ -z "''${repair_project_dir:-}" ]; then - continue - fi - if [ ! -d "$repair_project_dir" ] || [ ! -f "$repair_project_dir/pnpm-lock.yaml" ]; then - echo "[pnpm] Skipping stale dependency materialization root: $repair_project_dir" >&2 - continue - fi - - mkdir -p "$repair_store_dir" - repair_store_lockfile="$repair_store_dir/.effect-utils-pnpm-store.lock" - exec 204>"$repair_store_lockfile" - if ! ${flock} -w 600 204; then - echo "[pnpm] store-dir repair lock timeout after 600s: $repair_store_lockfile" >&2 - exit 1 - fi - - echo "[pnpm] Repairing dependency materialization root: $repair_project_dir" - ( - cd "$repair_project_dir" - export PNPM_STORE_DIR="$repair_store_dir" - export PNPM_CONFIG_STORE_DIR="$repair_store_dir" - export npm_config_store_dir="$repair_store_dir" - ${runPnpmInstallFn} - run_pnpm_install --force - ) - repaired_roots=$((repaired_roots + 1)) - done < <(dependency_materialization_repair_roots ${pkgs.nodejs}/bin/node "$repair_registry_file" "$files_pool_id") - - if [ "$repaired_roots" -eq 0 ]; then - echo "[pnpm] No live dependency materialization roots were repaired" >&2 + if [ -n "''${CI:-}" ]; then + echo "[pnpm] Legacy host Store Cache migration is unavailable in CI" >&2 exit 1 fi + ${loadPnpmTaskHelpersFn} + store_dir="''${PNPM_SHARED_STORE_DIR:-$HOME/.local/share/pnpm/store-shared-v1}" + expected_legacy_files="$HOME/.local/share/pnpm/shared-files/v11" + acquire_pnpm_store_cache_lease ${lib.escapeShellArg flock} exclusive "$store_dir" 600 + migrate_legacy_pnpm_store_cache "$store_dir" "$expected_legacy_files" ''; }; @@ -931,21 +779,10 @@ assert lib.assertMsg pnpmLockMutatorOverrideIsSupported '' enterShell = lib.mkIf (globalCache && workspaceRoot == ".") '' export PNPM_HOME="''${PNPM_HOME:-${config.devenv.root}/.devenv/pnpm-home}" - _pnpm_store_dir="''${npm_config_store_dir:-''${PNPM_CONFIG_STORE_DIR:-''${PNPM_STORE_DIR:-${defaultPnpmStoreDir}}}}" - export PNPM_STORE_DIR="$_pnpm_store_dir" - export PNPM_CONFIG_STORE_DIR="$_pnpm_store_dir" - export npm_config_store_dir="$_pnpm_store_dir" + source ${lib.escapeShellArg pnpmTaskHelpersScript} + ${configurePnpmStorageFn} export npm_config_cache="$HOME/.cache/pnpm" export npm_config_pm_on_fail=ignore - if [ -z "''${CI:-}" ]; then - _pnpm_shared_files="''${PNPM_SHARED_FILES_DIR:-$HOME/.local/share/pnpm/shared-files}/v11" - mkdir -p "$PNPM_STORE_DIR/v11" "$_pnpm_shared_files" - if [ ! -e "$PNPM_STORE_DIR/v11/files" ] && [ ! -L "$PNPM_STORE_DIR/v11/files" ]; then - ln -s "$_pnpm_shared_files" "$PNPM_STORE_DIR/v11/files" - fi - unset _pnpm_shared_files - fi - unset _pnpm_store_dir ''; tasks = cliGuard.stripGuards allTasks; diff --git a/nix/devenv-modules/tasks/shared/tests/pnpm-shared-store-reuse.test.sh b/nix/devenv-modules/tasks/shared/tests/pnpm-shared-store-reuse.test.sh new file mode 100644 index 0000000000..a2589047d9 --- /dev/null +++ b/nix/devenv-modules/tasks/shared/tests/pnpm-shared-store-reuse.test.sh @@ -0,0 +1,292 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../../../../.." && pwd)" + +PNPM_OUT="$({ + nix build --no-link --print-out-paths --impure --expr " + let + flake = builtins.getFlake (toString $ROOT); + pkgs = import flake.inputs.nixpkgs { system = builtins.currentSystem; }; + in flake.lib.mkPnpm { inherit pkgs; } + " +})" +PNPM="$PNPM_OUT/bin/pnpm" +EXPECTED_PNPM_VERSION="$(node -e 'const packageManager=require(process.argv[1]).packageManager; process.stdout.write(packageManager.slice(packageManager.lastIndexOf("@") + 1))' "$ROOT/package.json")" +ACTUAL_PNPM_VERSION="$($PNPM --version)" +if [ "$ACTUAL_PNPM_VERSION" != "$EXPECTED_PNPM_VERSION" ]; then + echo "FAIL: shared-store proof pnpm version $ACTUAL_PNPM_VERSION != workspace authority $EXPECTED_PNPM_VERSION" >&2 + exit 1 +fi + +assert_contains() { + local file="$1" + local pattern="$2" + local label="$3" + + if ! grep -Eq "$pattern" "$file"; then + echo "FAIL: $label" >&2 + sed -n '1,160p' "$file" >&2 + exit 1 + fi +} + +run_pnpm_logged() { + local workspace_root="$1" + local log_file="$2" + shift 2 + + local rc + set +e + if [ "$(uname -s)" = Darwin ]; then + NODE_OPTIONS="${NODE_OPTIONS:+$NODE_OPTIONS }--max-old-space-size=1536" "$@" > "$log_file" 2>&1 + else + "$@" > "$log_file" 2>&1 + fi + rc="$?" + set -e + + if [ "$rc" -eq 0 ]; then + return + fi + + # Mirror the production install policy: pnpm/Node can abort during Darwin + # teardown after completing materialization. Normalize only the exact abort + # with both pnpm completion evidence and a complete root-local projection. + if [ "$rc" -eq 134 ] && [ "$(uname -s)" = Darwin ] && + grep -qE 'Progress: .* done$' "$log_file" && + [ -d "$workspace_root/node_modules/.pnpm" ] && + [ -f "$workspace_root/node_modules/.modules.yaml" ]; then + return + fi + + cat "$log_file" >&2 + return "$rc" +} + +inode_id() { + node -e 'const fs=require("node:fs"); const s=fs.statSync(process.argv[1]); process.stdout.write(`${s.dev}:${s.ino}`)' "$1" +} + +real_path() { + node -e 'const fs=require("node:fs"); process.stdout.write(fs.realpathSync(process.argv[1]))' "$1" +} + +tmpdir="$(mktemp -d)" +trap 'rm -rf "$tmpdir"' EXIT + +mkdir -p \ + "$tmpdir/immutable-package/package" \ + "$tmpdir/native-package/package" \ + "$tmpdir/root-a" \ + "$tmpdir/root-b" \ + "$tmpdir/shared-store" + +# Ordinary immutable package data may be cloned, copied, or hardlinked from the +# shared store. The selected mechanism is evidence, not package identity. +cat > "$tmpdir/immutable-package/package/package.json" <<'EOF' +{ + "name": "hardlink-proof", + "version": "1.0.0", + "files": ["index.js"] +} +EOF +printf 'module.exports = "immutable"\n' > "$tmpdir/immutable-package/package/index.js" +tar -czf "$tmpdir/hardlink-proof-1.0.0.tgz" -C "$tmpdir/immutable-package" package + +# A native/source-build package may mutate its package directory. pnpm must +# keep its install hook pending and import its files on distinct inodes. +cat > "$tmpdir/native-package/package/package.json" <<'EOF' +{ + "name": "native-mutator", + "version": "1.0.0", + "files": ["index.js", "install.cjs"], + "scripts": {"install": "node install.cjs"} +} +EOF +printf 'module.exports = "native-original"\n' > "$tmpdir/native-package/package/index.js" +cat > "$tmpdir/native-package/package/install.cjs" <<'EOF' +const fs = require("node:fs") +const path = require("node:path") +fs.writeFileSync(path.join(__dirname, "index.js"), "module.exports = \"mutated\"\n") +fs.writeFileSync(path.join(__dirname, "install-ran"), "unsafe\n") +EOF +tar -czf "$tmpdir/native-mutator-1.0.0.tgz" -C "$tmpdir/native-package" package + +for root_name in root-a root-b; do + cat > "$tmpdir/$root_name/package.json" <&2 + cat "$tmpdir/root-b.log" >&2 + exit 1 +fi + +test -f "$tmpdir/shared-store/v11/index.db" +test "$(real_path "$tmpdir/root-a/node_modules/.pnpm")" = "$(real_path "$tmpdir/root-a")/node_modules/.pnpm" +test "$(real_path "$tmpdir/root-b/node_modules/.pnpm")" = "$(real_path "$tmpdir/root-b")/node_modules/.pnpm" +test "$(real_path "$tmpdir/root-a/node_modules/.pnpm")" != "$(real_path "$tmpdir/root-b/node_modules/.pnpm")" + +root_a_file="$tmpdir/root-a/node_modules/hardlink-proof/index.js" +root_b_file="$tmpdir/root-b/node_modules/hardlink-proof/index.js" +ordinary_inode_shared=false +if [ "$(inode_id "$root_a_file")" = "$(inode_id "$root_b_file")" ]; then + ordinary_inode_shared=true +fi +if [ "$(uname -s)" = Linux ] && [ "$ordinary_inode_shared" != true ]; then + reflink_probe="$tmpdir/reflink-probe" + if ! cp --reflink=always "$root_a_file" "$reflink_probe" 2>/dev/null; then + echo "FAIL: Linux auto import neither shared an inode nor ran on a reflink-capable filesystem" >&2 + exit 1 + fi +fi +test "$(cat "$root_a_file")" = 'module.exports = "immutable"' + +native_a_file="$tmpdir/root-a/node_modules/native-mutator/index.js" +native_b_file="$tmpdir/root-b/node_modules/native-mutator/index.js" +if [ "$(inode_id "$native_a_file")" = "$(inode_id "$native_b_file")" ]; then + echo "FAIL: requires-build package data shares a mutable hardlink inode" >&2 + exit 1 +fi +test "$(cat "$native_a_file")" = 'module.exports = "native-original"' +test "$(cat "$native_b_file")" = 'module.exports = "native-original"' +test ! -e "$tmpdir/root-a/node_modules/native-mutator/install-ran" +test ! -e "$tmpdir/root-b/node_modules/native-mutator/install-ran" + +node - "$tmpdir/root-a/node_modules/.modules.yaml" <<'EOF' +const fs = require("node:fs") +const modules = JSON.parse(fs.readFileSync(process.argv[2], "utf8")) +if (!modules.pendingBuilds.some((entry) => entry.startsWith("native-mutator@"))) { + throw new Error("native/source-build mutation was not retained as an explicit pending build") +} +if (modules.virtualStoreDir !== ".pnpm") { + throw new Error(`expected root-local virtual store, got ${modules.virtualStoreDir}`) +} +EOF + +rm -rf "$tmpdir/root-b/node_modules" +run_pnpm_logged "$tmpdir/root-b" "$tmpdir/root-b-offline.log" "$PNPM" --dir "$tmpdir/root-b" install --offline --force --frozen-lockfile "${install_flags[@]}" +assert_contains \ + "$tmpdir/root-b-offline.log" \ + 'Progress: resolved [0-9]+, reused [1-9][0-9]*, downloaded 0, added [1-9][0-9]*, done' \ + "offline rematerialization must reuse the shared full store" + +# A full shared store has one writable package index. Prove pnpm itself can own +# that concurrency boundary: start two roots against a second empty store, then +# require both independently selected graphs to survive offline rematerialization. +mkdir -p "$tmpdir/concurrent-a" "$tmpdir/concurrent-b" "$tmpdir/concurrent-store" +for root_name in concurrent-a concurrent-b; do + cat > "$tmpdir/$root_name/package.json" <&2 + cat "$tmpdir/concurrent-a.log" >&2 + exit 1 +fi +if ! wait "$concurrent_b_pid"; then + echo "FAIL: concurrent root B cold install" >&2 + cat "$tmpdir/concurrent-b.log" >&2 + exit 1 +fi + +test -f "$tmpdir/concurrent-store/v11/index.db" +test "$(real_path "$tmpdir/concurrent-a/node_modules/.pnpm")" != "$(real_path "$tmpdir/concurrent-b/node_modules/.pnpm")" +for root_name in concurrent-a concurrent-b; do + test "$(cat "$tmpdir/$root_name/node_modules/hardlink-proof/index.js")" = 'module.exports = "immutable"' + test "$(cat "$tmpdir/$root_name/node_modules/native-mutator/index.js")" = 'module.exports = "native-original"' + test ! -e "$tmpdir/$root_name/node_modules/native-mutator/install-ran" + node -e 'if (require(process.argv[1]) !== "immutable") process.exit(1)' \ + "$tmpdir/$root_name/node_modules/hardlink-proof" +done + +rm -rf "$tmpdir/concurrent-a/node_modules" "$tmpdir/concurrent-b/node_modules" +run_pnpm_logged "$tmpdir/concurrent-a" "$tmpdir/concurrent-a-offline.log" "$PNPM" --dir "$tmpdir/concurrent-a" install --offline --force "${concurrent_flags[@]}" & +concurrent_a_pid=$! +run_pnpm_logged "$tmpdir/concurrent-b" "$tmpdir/concurrent-b-offline.log" "$PNPM" --dir "$tmpdir/concurrent-b" install --offline --force "${concurrent_flags[@]}" & +concurrent_b_pid=$! +if ! wait "$concurrent_a_pid"; then + echo "FAIL: concurrent root A offline rematerialization" >&2 + cat "$tmpdir/concurrent-a-offline.log" >&2 + exit 1 +fi +if ! wait "$concurrent_b_pid"; then + echo "FAIL: concurrent root B offline rematerialization" >&2 + cat "$tmpdir/concurrent-b-offline.log" >&2 + exit 1 +fi +for root_name in concurrent-a concurrent-b; do + test "$(cat "$tmpdir/$root_name/node_modules/hardlink-proof/index.js")" = 'module.exports = "immutable"' + test "$(cat "$tmpdir/$root_name/node_modules/native-mutator/index.js")" = 'module.exports = "native-original"' + test ! -e "$tmpdir/$root_name/node_modules/native-mutator/install-ran" +done + +# Direct package-file mutation is outside the managed materialization contract, +# but an inode alias would make its blast radius cross-root. Measure that risk +# explicitly without making hardlinking itself a required outcome. +chmod u+w "$root_a_file" +printf 'module.exports = "mutation-probe"\n' > "$root_a_file" +mutation_aliased=false +if grep -q 'mutation-probe' "$root_b_file"; then + mutation_aliased=true +fi +if [ "$ordinary_inode_shared" != "$mutation_aliased" ]; then + echo "FAIL: inode-sharing and mutation-alias evidence disagree" >&2 + exit 1 +fi +test "$(cat "$native_b_file")" = 'module.exports = "native-original"' + +printf '{"phase":"shared-store-reuse","status":"ok","secondRootDownloads":0,"ordinaryInodeShared":%s,"mutationAliased":%s,"nativeInodeDistinct":true,"virtualStoresDistinct":true,"concurrentColdRoots":2,"concurrentOfflineRoots":2,"sharedIndexHealthy":true,"lifecycleHooksRan":0}\n' \ + "$ordinary_inode_shared" \ + "$mutation_aliased" diff --git a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh index 048e6ba7c6..3efda825c5 100644 --- a/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/pnpm-task-smoke.test.sh @@ -198,7 +198,7 @@ echo "Running pnpm task smoke test..." echo "" tmpdir="$(mktemp -d)" -trap 'rm -rf "$tmpdir"' EXIT +trap 'if [ "${KEEP_PNPM_SMOKE_TMP:-0}" = "1" ]; then echo "pnpm smoke tmp: $tmpdir" >&2; else rm -rf "$tmpdir"; fi' EXIT workspace="$tmpdir/workspace" mkdir -p "$workspace/.devenv/task-cache" "$workspace/.pnpm-home-a/store/v11" "$workspace/.pnpm-home-b/store/v11" "$tmpdir/bin" "$workspace/packages/demo/node_modules/.bin" "$workspace/nested/pkg" @@ -222,36 +222,10 @@ cat > "$workspace/pnpm-install-contract.json" <<'EOF' { "schemaVersion": 1, "packageManager": {"name": "pnpm", "version": "11.3.0"}, - "gvsLinkContract": {"allowBuilds": {}, "packageExtensions": {}, "packageManager": {"name": "pnpm", "version": "11.3.0"}}, + "dependencyGraphContract": {"allowBuilds": {}, "packageExtensions": {}, "packageManager": {"name": "pnpm", "version": "11.3.0"}, "virtualStore": {"scope": "materialization-root", "path": "node_modules/.pnpm"}}, "installPolicy": {"ignoreScripts": true}, "storeContract": {"layoutVersion": "v11", "owner": "pnpm", "storeDir": ".devenv/pnpm-store-pure-v1"}, - "workspaceManifestContract": {"packages": []}, - "dependencyMaterializationProfile": { - "schema": "dependency-materialization-profile/v0", - "identityInputs": ["packageManager", "gvsLinkContract", "installPolicy", "storeContract", "workspaceManifestContract"], - "supportedTraits": { - "ciJobLocal": { - "mutableState": "job-local", - "gcAuthority": "profile-local", - "repairAuthority": "ci-job" - }, - "darwinSplitCas": { - "mutableState": "profile-local", - "sharedContent": "store/v11/files", - "gcAuthority": "shared-pool-coordinator", - "repairAuthority": "devenv" - }, - "isolated": { - "mutableState": "profile-local", - "gcAuthority": "profile-local", - "repairAuthority": "devenv" - } - }, - "nativeBuildPolicyInputs": { - "allowBuilds": "gvsLinkContract.allowBuilds", - "compilerEnv": ["CC", "CXX"] - } - } + "workspaceManifestContract": {"packages": []} } EOF chmod 444 "$workspace/pnpm-install-contract.json" @@ -295,17 +269,23 @@ if [ "${1:-}" = "install" ]; then echo "ERR_PNPM_META_FETCH_FAIL GET https://registry.npmjs.org/demo: request to https://registry.npmjs.org/demo failed, reason: Socket timeout" >&2 exit 42 fi - mkdir -p node_modules/.pnpm vendor/pkg-v1 + mkdir -p \ + node_modules/.pnpm/pkg@1.0.0/node_modules/pkg/node_modules \ + node_modules/.pnpm/dep@1.0.0/node_modules/dep \ + vendor/foreign-root/node_modules/.pnpm/dep@2.0.0/node_modules/dep touch node_modules/.install-ok - printf '{"name":"pkg","version":"1.0.0"}\n' > vendor/pkg-v1/package.json - ln -snf ../vendor/pkg-v1 node_modules/pkg + printf '{"name":"pkg","version":"1.0.0","dependencies":{"dep":"1.0.0"}}\n' > node_modules/.pnpm/pkg@1.0.0/node_modules/pkg/package.json + printf '{"name":"dep","version":"1.0.0"}\n' > node_modules/.pnpm/dep@1.0.0/node_modules/dep/package.json + printf '{"name":"dep","version":"2.0.0"}\n' > vendor/foreign-root/node_modules/.pnpm/dep@2.0.0/node_modules/dep/package.json + ln -snf .pnpm/pkg@1.0.0/node_modules/pkg node_modules/pkg + ln -snf ../../../../dep@1.0.0/node_modules/dep node_modules/.pnpm/pkg@1.0.0/node_modules/pkg/node_modules/dep # The warm-path status now fingerprints the root projection metadata that # pnpm always writes on a real install. Keep the smoke fixture aligned with # that contract so the test still exercises the task logic instead of # failing on an unrealistically incomplete fake install. cat > node_modules/.modules.yaml <&2 exit 1 EOF chmod +x "$tmpdir/bin/pnpm" +cat > "$tmpdir/bin/devenv" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +if [ "$*" != "tasks run pnpm:install" ]; then + echo "unexpected fake devenv invocation: $*" >&2 + exit 1 +fi +exec bash "${TEST_INSTALL_SCRIPT:?}" +EOF +chmod +x "$tmpdir/bin/devenv" + cat > "$tmpdir/bin/pnpm-lock-mutator" <<'EOF' #!/usr/bin/env bash set -euo pipefail @@ -388,13 +382,13 @@ printf 'storybook-shim:%s\n' "$*" EOF chmod +x "$workspace/packages/demo/node_modules/.bin/storybook" -extract_task_script "$workspace" "exec" "$tmpdir/pnpm-install.exec.sh" -extract_task_script "$workspace" "status" "$tmpdir/pnpm-install.status.sh" +extract_task_script "$workspace" "exec" "$tmpdir/pnpm-install.exec.sh" 'packages = [ ]; postInstallProjection = "touch .post-install-projection-marker";' +extract_task_script "$workspace" "status" "$tmpdir/pnpm-install.status.sh" 'packages = [ ]; postInstallProjection = "touch .post-install-projection-marker";' extract_task_script "$workspace" "exec" "$tmpdir/pnpm-doctor.exec.sh" 'packages = [ ];' "pnpm:doctor" -extract_task_script "$workspace" "exec" "$tmpdir/pnpm-repair-plan.exec.sh" 'packages = [ ];' "pnpm:repair-plan" extract_task_script "$workspace" "exec" "$tmpdir/pnpm-repair.exec.sh" 'packages = [ ];' "pnpm:repair" extract_task_script "$workspace" "exec" "$tmpdir/pnpm-clean.exec.sh" 'packages = [ "packages/demo" ];' "pnpm:clean" extract_task_script "$workspace" "exec" "$tmpdir/pnpm-update.exec.sh" 'packages = [ ];' "pnpm:update" +extract_task_script "$workspace" "exec" "$tmpdir/pnpm-dedupe.exec.sh" 'packages = [ ];' "pnpm:dedupe" extract_task_script "$workspace" "exec" "$tmpdir/pnpm-update-nested.exec.sh" 'packages = [ ]; workspaceRoot = "nested"; taskSuffix = "nested";' "pnpm:update:nested" extract_task_script "$workspace" "exec" "$tmpdir/pnpm-install-nested.exec.sh" 'packages = [ "pkg" ]; workspaceRoot = "nested"; taskSuffix = "nested";' "pnpm:install:nested" extract_task_script "$workspace" "status" "$tmpdir/pnpm-install-nested.status.sh" 'packages = [ "pkg" ]; workspaceRoot = "nested"; taskSuffix = "nested";' "pnpm:install:nested" @@ -408,6 +402,7 @@ extract_task_script "$workspace" "exec" "$tmpdir/pnpm-install-impure-strict-stor extract_task_script "$workspace" "exec" "$tmpdir/pnpm-install-impure-pm-on-fail.exec.sh" 'packages = [ "." ]; installFlags = [ "--pm-on-fail=download" ];' "pnpm:install" extract_task_script "$workspace" "exec" "$tmpdir/pnpm-install-impure-ignore-scripts.exec.sh" 'packages = [ "." ]; installFlags = [ "--config.ignore-scripts=false" ];' "pnpm:install" extract_task_script "$workspace" "exec" "$tmpdir/pnpm-install-impure-ignore-dep-scripts.exec.sh" 'packages = [ "." ]; installFlags = [ "--config.ignore-dep-scripts=false" ];' "pnpm:install" +extract_task_script "$workspace" "exec" "$tmpdir/pnpm-install-impure-gvs.exec.sh" 'packages = [ "." ]; installFlags = [ "--config.enable-global-virtual-store=true" ];' "pnpm:install" extract_shared_task_script \ "nix/devenv-modules/tasks/shared/test.nix" \ "test:demo" \ @@ -423,10 +418,10 @@ extract_shared_task_script \ rewrite_unrealized_tool_paths "$tmpdir/pnpm-install.exec.sh" rewrite_unrealized_tool_paths "$tmpdir/pnpm-install.status.sh" rewrite_unrealized_tool_paths "$tmpdir/pnpm-doctor.exec.sh" -rewrite_unrealized_tool_paths "$tmpdir/pnpm-repair-plan.exec.sh" rewrite_unrealized_tool_paths "$tmpdir/pnpm-repair.exec.sh" rewrite_unrealized_tool_paths "$tmpdir/pnpm-clean.exec.sh" rewrite_unrealized_tool_paths "$tmpdir/pnpm-update.exec.sh" +rewrite_unrealized_tool_paths "$tmpdir/pnpm-dedupe.exec.sh" rewrite_unrealized_tool_paths "$tmpdir/pnpm-update-nested.exec.sh" rewrite_unrealized_tool_paths "$tmpdir/pnpm-install-nested.exec.sh" rewrite_unrealized_tool_paths "$tmpdir/pnpm-install-nested.status.sh" @@ -439,12 +434,14 @@ rewrite_unrealized_tool_paths "$tmpdir/pnpm-install-impure-strict-store.exec.sh" rewrite_unrealized_tool_paths "$tmpdir/pnpm-install-impure-pm-on-fail.exec.sh" rewrite_unrealized_tool_paths "$tmpdir/pnpm-install-impure-ignore-scripts.exec.sh" rewrite_unrealized_tool_paths "$tmpdir/pnpm-install-impure-ignore-dep-scripts.exec.sh" +rewrite_unrealized_tool_paths "$tmpdir/pnpm-install-impure-gvs.exec.sh" rewrite_unrealized_tool_paths "$tmpdir/test-demo.exec.sh" rewrite_unrealized_tool_paths "$tmpdir/storybook-demo.exec.sh" export PATH="$tmpdir/bin:$PATH" export TEST_PNPM_LOG="$tmpdir/pnpm.log" export TEST_FLOCK_LOG="$tmpdir/flock.log" +export TEST_INSTALL_SCRIPT="$tmpdir/pnpm-install.exec.sh" export TEST_PNPM_MUTATOR_LOG="$tmpdir/pnpm-mutator.log" export TEST_GENIE_LOG="$tmpdir/genie.log" unset CI @@ -452,6 +449,21 @@ unset PNPM_STORE_DIR unset PNPM_CONFIG_STORE_DIR unset npm_config_store_dir +echo "Test 0: install ignores disposable historical root-local cache content" +( + cd "$workspace" + export HOME="$tmpdir/home" + unset PNPM_HOME + legacy_files="$workspace/.devenv/pnpm-store-pure-v1/v11/files" + mkdir -p "$legacy_files" + printf 'preserve-me\n' > "$legacy_files/sentinel" + bash "$tmpdir/pnpm-install.exec.sh" + test -f "$legacy_files/sentinel" + test ! -L "$legacy_files" + rm -rf "$legacy_files" + rm -rf "$workspace/node_modules" "$workspace/vendor" "$workspace/.devenv/task-cache/pnpm-install" +) + echo "Test 1: status misses before install" ( cd "$workspace" @@ -473,14 +485,18 @@ echo "Test 2: exec runs fake pnpm and populates cache" bash "$tmpdir/pnpm-install.exec.sh" test -f "$workspace/.devenv/task-cache/pnpm-install/install-state.hash" test -f "$workspace/.devenv/task-cache/pnpm-install/projection-state.hash" + test -f "$workspace/.devenv/task-cache/pnpm-install/pnpm-storage-state" + test -f "$workspace/.post-install-projection-marker" test -d "$workspace/node_modules" test -f "$workspace/node_modules/.modules.yaml" grep -qxF "flock -w 600 200" "$tmpdir/flock.log" grep -qxF "flock -w 600 201" "$tmpdir/flock.log" - grep -qxF "flock -w 600 202" "$tmpdir/flock.log" - grep -qxF "install --force --frozen-lockfile --config.confirmModulesPurge=false --ignore-scripts --config.side-effects-cache=false --config.verify-store-integrity=true --config.strict-store-pkg-content-check=true --child-concurrency=1 --network-concurrency=4 --config.package-import-method=clone-or-copy --pm-on-fail=ignore --config.store-dir=$workspace/.devenv/pnpm-store-pure-v1" "$tmpdir/pnpm.log" + grep -qxF "flock --shared -w 600 202" "$tmpdir/flock.log" + ! grep -qF -- '--exclusive' "$tmpdir/flock.log" + test "$(wc -l < "$tmpdir/flock.log")" -eq 3 + grep -qxF "install --frozen-lockfile --config.confirmModulesPurge=false --ignore-scripts --config.side-effects-cache=false --config.verify-store-integrity=true --config.strict-store-pkg-content-check=true --child-concurrency=1 --network-concurrency=4 --config.enable-global-virtual-store=false --config.virtual-store-dir=node_modules/.pnpm --pm-on-fail=ignore --config.package-import-method=auto --config.store-dir=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm.log" grep -qF ".effect-utils-pnpm-install.lock" "$tmpdir/pnpm-install.exec.sh" - grep -qF ".effect-utils-pnpm-store.lock" "$tmpdir/pnpm-install.exec.sh" + ! grep -qF ".effect-utils-pnpm-store.lock" "$tmpdir/pnpm-install.exec.sh" test -w "$workspace/.devenv/task-cache/pnpm-install/pnpm-install-contract.json" ) @@ -494,7 +510,31 @@ echo "Test 2b: exec replaces a read-only cached generated contract snapshot" test -w "$workspace/.devenv/task-cache/pnpm-install/pnpm-install-contract.json" ) -echo "Test 3: status hits after install with same GVS path" +echo "Test 2c: lockfile mutation entrypoints preserve the live topology policy" +( + cd "$workspace" + export HOME="$tmpdir/home" + export PNPM_HOME="$workspace/.pnpm-home-a" + : > "$tmpdir/pnpm.log" + : > "$tmpdir/pnpm-mutator.log" + : > "$tmpdir/flock.log" + bash "$tmpdir/pnpm-update.exec.sh" + bash "$tmpdir/pnpm-dedupe.exec.sh" + policy_flags="--config.confirmModulesPurge=false --ignore-scripts --config.side-effects-cache=false --config.verify-store-integrity=true --config.strict-store-pkg-content-check=true --child-concurrency=1 --network-concurrency=4 --config.enable-global-virtual-store=false --config.virtual-store-dir=node_modules/.pnpm --pm-on-fail=ignore" + grep -qxF "install --fix-lockfile $policy_flags --config.package-import-method=auto --config.store-dir=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm-mutator.log" + grep -qxF "dedupe $policy_flags --config.package-import-method=auto --config.store-dir=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm.log" + test "$(grep -cFx 'flock -w 600 200' "$tmpdir/flock.log")" -eq 2 + test "$(grep -cFx 'flock -w 600 201' "$tmpdir/flock.log")" -eq 2 + test "$(grep -cFx 'flock --shared -w 600 202' "$tmpdir/flock.log")" -eq 2 + ! grep -qF -- '--exclusive' "$tmpdir/flock.log" + test "$(wc -l < "$tmpdir/flock.log")" -eq 6 + ! grep -qF "migrate_legacy_pnpm_store" "$tmpdir/pnpm-update.exec.sh" + grep -qF "assert_pnpm_storage_capacity" "$tmpdir/pnpm-update.exec.sh" + ! grep -qF "migrate_legacy_pnpm_store" "$tmpdir/pnpm-dedupe.exec.sh" + grep -qF "assert_pnpm_storage_capacity" "$tmpdir/pnpm-dedupe.exec.sh" +) + +echo "Test 3: status hits after install with the same root-local virtual topology" ( cd "$workspace" export HOME="$tmpdir/home" @@ -506,6 +546,22 @@ echo "Test 3: status hits after install with same GVS path" assert_exit_code 0 "$exit_code" "status should hit after install" ) +echo "Test 3b: cached status rejects a nested dependency edge outside the root-local topology" +( + cd "$workspace" + export HOME="$tmpdir/home" + export PNPM_HOME="$workspace/.pnpm-home-a" + export DEVENV_SETUP_OUTER_CACHE_HIT=1 + ln -snf "$workspace/vendor/foreign-root/node_modules/.pnpm/dep@2.0.0/node_modules/dep" node_modules/.pnpm/pkg@1.0.0/node_modules/pkg/node_modules/dep + set +e + bash "$tmpdir/pnpm-install.status.sh" + exit_code=$? + set -e + unset DEVENV_SETUP_OUTER_CACHE_HIT + assert_exit_code 1 "$exit_code" "cached status should reject a foreign nested dependency edge" + ln -snf ../../../../dep@1.0.0/node_modules/dep node_modules/.pnpm/pkg@1.0.0/node_modules/pkg/node_modules/dep +) + echo "Test 4: outer cache hit still misses when projection metadata is missing" ( cd "$workspace" @@ -555,41 +611,29 @@ echo "Test 7: exec defaults PNPM_HOME to a workspace-local projection" : > "$tmpdir/pnpm.log" bash "$tmpdir/pnpm-install.exec.sh" grep -qxF "PNPM_HOME=$workspace/.devenv/pnpm-home" "$tmpdir/pnpm.log" - grep -qxF "PNPM_STORE_DIR=$workspace/.devenv/pnpm-store-pure-v1" "$tmpdir/pnpm.log" - grep -qxF "PNPM_CONFIG_STORE_DIR=$workspace/.devenv/pnpm-store-pure-v1" "$tmpdir/pnpm.log" - grep -qxF "npm_config_store_dir=$workspace/.devenv/pnpm-store-pure-v1" "$tmpdir/pnpm.log" - test -L "$workspace/.devenv/pnpm-store-pure-v1/v11/files" - test "$(readlink "$workspace/.devenv/pnpm-store-pure-v1/v11/files")" = "$tmpdir/home/.local/share/pnpm/shared-files/v11" - profile_file="$workspace/.devenv/task-cache/pnpm-install/dependency-materialization-profile.json" - registry_file="$workspace/.devenv/task-cache/pnpm-install/dependency-materialization-registry.json" - test -f "$profile_file" - test -f "$registry_file" - profile_id="$(node -e 'const fs=require("node:fs"); process.stdout.write(JSON.parse(fs.readFileSync(process.argv[1],"utf8")).profileId)' "$profile_file")" - assert_json_field "dependency-materialization-profile/v0" "$profile_file" "value => value.schema" "live profile schema" - assert_json_field "true" "$profile_file" "value => value.profileId.startsWith('pnpm:')" "live profile id prefix" - assert_json_field "darwinSplitCas" "$profile_file" "value => value.store.trait" "live profile split store trait" - assert_json_field "shared-pool-coordinator" "$profile_file" "value => value.authorities.gc" "live profile gc authority" - assert_json_field "pnpm-install-contract.json" "$profile_file" "value => value.evidence.contractPath" "live profile relative contract path" - assert_json_field "dependency-materialization-registry/v0" "$registry_file" "value => value.schema" "live registry schema" - assert_json_field "$profile_id" "$registry_file" "value => value.profiles[0].profileId" "live registry profile id" - assert_json_field "$workspace" "$registry_file" "value => value.profiles[0].project" "live registry project" - assert_json_field "$workspace/.devenv/pnpm-store-pure-v1" "$registry_file" "value => value.profiles[0].store" "live registry store" - assert_json_field "$workspace/.devenv/pnpm-store-pure-v1/v11/files" "$registry_file" "value => value.pools[0].filesPath" "live registry files path" + grep -qxF "PNPM_STORE_DIR=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm.log" + grep -qxF "PNPM_CONFIG_STORE_DIR=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm.log" + grep -qxF "npm_config_store_dir=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm.log" + test -d "$tmpdir/home/.local/share/pnpm/store-shared-v1/v11/files" doctor_decision="$(bash "$tmpdir/pnpm-doctor.exec.sh" | node -e 'const fs=require("node:fs"); process.stdout.write(JSON.parse(fs.readFileSync(0,"utf8")).decision)')" - assert_eq "refuse-raw-prune" "$doctor_decision" "doctor refuses raw prune for shared files pool" - repair_decision="$(bash "$tmpdir/pnpm-repair-plan.exec.sh" | node -e 'const fs=require("node:fs"); const value=JSON.parse(fs.readFileSync(0,"utf8")); process.stdout.write(`${value.decision}:${value.roots.length}`)')" - assert_eq "repair-all-roots:1" "$repair_decision" "repair plan covers registered root" + assert_eq "healthy" "$doctor_decision" "doctor validates the root-local graph" + mkdir -p "$workspace/node_modules/corrupt-edge" + touch "$workspace/node_modules/corrupt-edge/sentinel" : > "$tmpdir/pnpm.log" - : > "$tmpdir/flock.log" bash "$tmpdir/pnpm-repair.exec.sh" >/dev/null - grep -qxF "install --force --frozen-lockfile --config.confirmModulesPurge=false --ignore-scripts --config.side-effects-cache=false --config.verify-store-integrity=true --config.strict-store-pkg-content-check=true --child-concurrency=1 --network-concurrency=4 --config.package-import-method=clone-or-copy --pm-on-fail=ignore --config.store-dir=$workspace/.devenv/pnpm-store-pure-v1" "$tmpdir/pnpm.log" + test ! -e "$workspace/node_modules/corrupt-edge" + test -f "$workspace/node_modules/.install-ok" grep -qxF "PWD=$workspace" "$tmpdir/pnpm.log" - grep -qxF "flock -w 600 204" "$tmpdir/flock.log" CI=1 bash "$tmpdir/pnpm-install.exec.sh" - assert_json_field "ciJobLocal" "$profile_file" "value => value.store.trait" "CI install records CI-local profile trait" + set +e + bash "$tmpdir/pnpm-install.status.sh" + exit_code=$? + set -e + assert_exit_code 1 "$exit_code" "local status should miss after a CI job-local store install" + bash "$tmpdir/pnpm-install.exec.sh" ) -echo "Test 8: status hits after install with the default GVS path" +echo "Test 8: status hits after install with the default root-local topology" ( cd "$workspace" export HOME="$tmpdir/home" @@ -630,7 +674,7 @@ echo "Test 10: status still hits when PNPM_HOME changes but store-dir stays shar assert_exit_code 0 "$exit_code" "status should hit when only PNPM_HOME changes" ) -echo "Test 11: status misses after effective store-dir changes" +echo "Test 11: ambient pnpm store variables do not split the canonical host store" ( cd "$workspace" export HOME="$tmpdir/home" @@ -641,13 +685,13 @@ echo "Test 11: status misses after effective store-dir changes" bash "$tmpdir/pnpm-install.status.sh" exit_code=$? set -e - assert_exit_code 1 "$exit_code" "status should miss when store-dir changes" + assert_exit_code 0 "$exit_code" "status should retain the canonical host store" ) echo "Test 12: exec invoked pnpm install" grep -q "^install " "$tmpdir/pnpm.log" -echo "Test 13: nested workspace exec uses its own cwd, cache, PNPM_HOME, and store-dir" +echo "Test 13: nested workspace exec uses its own cwd, cache, and PNPM_HOME with the host store" ( cd "$workspace" export HOME="$tmpdir/home" @@ -657,12 +701,13 @@ echo "Test 13: nested workspace exec uses its own cwd, cache, PNPM_HOME, and sto : > "$tmpdir/pnpm.log" bash "$tmpdir/pnpm-install-nested.exec.sh" test -f "$workspace/.devenv/task-cache/pnpm-install/nested/install-state.hash" + test ! -e "$workspace/.devenv/task-cache/pnpm-install/nested/pnpm-install-contract.json" test -d "$workspace/nested/node_modules" grep -qxF "PWD=$workspace/nested" "$tmpdir/pnpm.log" grep -qxF "PNPM_HOME=$workspace/.devenv/pnpm-home/nested" "$tmpdir/pnpm.log" - grep -qxF "PNPM_STORE_DIR=$workspace/.devenv/pnpm-store-pure-v1/nested" "$tmpdir/pnpm.log" - grep -qxF "PNPM_CONFIG_STORE_DIR=$workspace/.devenv/pnpm-store-pure-v1/nested" "$tmpdir/pnpm.log" - grep -qxF "npm_config_store_dir=$workspace/.devenv/pnpm-store-pure-v1/nested" "$tmpdir/pnpm.log" + grep -qxF "PNPM_STORE_DIR=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm.log" + grep -qxF "PNPM_CONFIG_STORE_DIR=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm.log" + grep -qxF "npm_config_store_dir=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm.log" ) echo "Test 14: nested workspace status hits after nested install" @@ -679,7 +724,24 @@ echo "Test 14: nested workspace status hits after nested install" assert_exit_code 0 "$exit_code" "nested status should hit after nested install" ) -echo "Test 15: nested workspace suffixes an inherited store-dir" +echo "Test 14b: root contract evidence is fail-closed and never inherited by nested roots" +( + cd "$workspace" + export HOME="$tmpdir/home" + export PNPM_HOME="$workspace/.pnpm-home-a" + mv pnpm-install-contract.json pnpm-install-contract.json.hidden + set +e + output="$(bash "$tmpdir/pnpm-install.status.sh" 2>&1)" + exit_code=$? + set -e + mv pnpm-install-contract.json.hidden pnpm-install-contract.json + assert_exit_code 1 "$exit_code" "repo root requires root-local generated contract evidence" + grep -qF "Missing generated pnpm-install-contract.json at repo root" <<< "$output" + + bash "$tmpdir/pnpm-install-nested.status.sh" +) + +echo "Test 15: nested workspace ignores ambient store-dir in favor of host authority" ( cd "$workspace" export HOME="$tmpdir/home" @@ -689,9 +751,9 @@ echo "Test 15: nested workspace suffixes an inherited store-dir" unset npm_config_store_dir : > "$tmpdir/pnpm.log" bash "$tmpdir/pnpm-install-nested.exec.sh" - grep -qxF "PNPM_STORE_DIR=$workspace/.inherited-pnpm-store/nested" "$tmpdir/pnpm.log" - grep -qxF "PNPM_CONFIG_STORE_DIR=$workspace/.inherited-pnpm-store/nested" "$tmpdir/pnpm.log" - grep -qxF "npm_config_store_dir=$workspace/.inherited-pnpm-store/nested" "$tmpdir/pnpm.log" + grep -qxF "PNPM_STORE_DIR=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm.log" + grep -qxF "PNPM_CONFIG_STORE_DIR=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm.log" + grep -qxF "npm_config_store_dir=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm.log" ) echo "Test 16: install flags and pre-install hooks are applied" @@ -705,7 +767,7 @@ echo "Test 16: install flags and pre-install hooks are applied" : > "$tmpdir/pnpm.log" bash "$tmpdir/pnpm-install-flags.exec.sh" test -f .preinstall-marker - grep -qxF "install --config.public-hoist-pattern=* --frozen-lockfile --config.confirmModulesPurge=false --ignore-scripts --config.side-effects-cache=false --config.verify-store-integrity=true --config.strict-store-pkg-content-check=true --child-concurrency=1 --network-concurrency=4 --config.package-import-method=clone-or-copy --pm-on-fail=ignore --config.store-dir=$workspace/.devenv/pnpm-store-pure-v1" "$tmpdir/pnpm.log" + grep -qxF "install --config.public-hoist-pattern=* --frozen-lockfile --config.confirmModulesPurge=false --ignore-scripts --config.side-effects-cache=false --config.verify-store-integrity=true --config.strict-store-pkg-content-check=true --child-concurrency=1 --network-concurrency=4 --config.enable-global-virtual-store=false --config.virtual-store-dir=node_modules/.pnpm --pm-on-fail=ignore --config.package-import-method=auto --config.store-dir=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm.log" ) echo "Test 17: impure no-frozen install flags are rejected before pnpm runs" @@ -852,6 +914,20 @@ echo "Test 24: impure ignore-dep-scripts overrides are rejected before pnpm runs fi ) +echo "Test 24b: shared writable virtual topology overrides are rejected before pnpm runs" +( + cd "$workspace" + export HOME="$tmpdir/home" + : > "$tmpdir/pnpm.log" + set +e + output="$(bash "$tmpdir/pnpm-install-impure-gvs.exec.sh" 2>&1)" + exit_code=$? + set -e + assert_exit_code 1 "$exit_code" "GVS override should be rejected" + grep -qF "Refusing impure install argument: --config.enable-global-virtual-store=true" <<< "$output" + test ! -s "$tmpdir/pnpm.log" +) + echo "Test 25: CI install failures preserve and classify the pnpm log" ( cd "$workspace" @@ -894,7 +970,7 @@ echo "Test 26: Darwin CI install accepts completed pnpm materialization after te test -f "$workspace/.devenv/task-cache/pnpm-install/install-state.hash" ) -echo "Test 27: Darwin CI install accepts completed pnpm materialization after teardown kill" +echo "Test 27: Darwin CI install rejects SIGKILL even after apparent materialization" ( cd "$workspace" export HOME="$tmpdir/home" @@ -905,11 +981,15 @@ echo "Test 27: Darwin CI install accepts completed pnpm materialization after te unset PNPM_STORE_DIR unset npm_config_store_dir rm -f "$workspace/.devenv/task-cache/pnpm-install/install-state.hash" + set +e output="$(bash "$tmpdir/pnpm-install-darwin.exec.sh" 2>&1)" + exit_code=$? + set -e unset TEST_PNPM_DARWIN_TEARDOWN_STATUS unset CI - grep -qF "[pnpm] Install completed materialization before darwin install teardown; continuing after node teardown exit 137" <<< "$output" - test -f "$workspace/.devenv/task-cache/pnpm-install/install-state.hash" + assert_exit_code 137 "$exit_code" "SIGKILL must not be promoted to a successful shared-store install" + grep -qF "[pnpm] Install failed: pnpm install failure" <<< "$output" + test ! -f "$workspace/.devenv/task-cache/pnpm-install/install-state.hash" ) echo "Test 28: generated test task runs vitest without pnpm exec" @@ -935,14 +1015,14 @@ echo "Test 29: generated storybook task runs storybook without pnpm exec" [ "$output" = "storybook-shim:build" ] ) -echo "Test 30: clean leaves shared GVS links intact" +echo "Test 30: clean removes only root-owned topology and leaves shared content intact" ( cd "$workspace" export HOME="$tmpdir/home" - mkdir -p "$workspace/.devenv/pnpm-store-pure-v1/v11/links/shared-pkg" + mkdir -p "$tmpdir/home/.local/share/pnpm/store-shared-v1/v11/files/shared-pkg" mkdir -p "$workspace/node_modules" "$workspace/packages/demo/node_modules" bash "$tmpdir/pnpm-clean.exec.sh" - test -d "$workspace/.devenv/pnpm-store-pure-v1/v11/links/shared-pkg" + test -d "$tmpdir/home/.local/share/pnpm/store-shared-v1/v11/files/shared-pkg" test ! -e "$workspace/node_modules" test ! -e "$workspace/packages/demo/node_modules" ) @@ -994,7 +1074,8 @@ EOF bash "$tmpdir/pnpm-update.exec.sh" grep -qxF -- "--defer-validation" "$tmpdir/genie.log" grep -qxF -- "--check" "$tmpdir/genie.log" - grep -qxF "install --fix-lockfile --config.confirmModulesPurge=false --pm-on-fail=ignore --config.store-dir=$workspace/.devenv/pnpm-store-pure-v1" "$tmpdir/pnpm-mutator.log" + policy_flags="--config.confirmModulesPurge=false --ignore-scripts --config.side-effects-cache=false --config.verify-store-integrity=true --config.strict-store-pkg-content-check=true --child-concurrency=1 --network-concurrency=4 --config.enable-global-virtual-store=false --config.virtual-store-dir=node_modules/.pnpm --pm-on-fail=ignore" + grep -qxF "install --fix-lockfile $policy_flags --config.package-import-method=auto --config.store-dir=$tmpdir/home/.local/share/pnpm/store-shared-v1" "$tmpdir/pnpm-mutator.log" grep -qF "hasBin: true" pnpm-lock.yaml ) diff --git a/nix/devenv-modules/tasks/shared/tests/pnpm.test.sh b/nix/devenv-modules/tasks/shared/tests/pnpm.test.sh index 5e0c35ae1d..2d777eb177 100644 --- a/nix/devenv-modules/tasks/shared/tests/pnpm.test.sh +++ b/nix/devenv-modules/tasks/shared/tests/pnpm.test.sh @@ -51,14 +51,14 @@ make_projection_fixture() { local root="$1" local with_dep="$2" local dep_blocks_package_json_export="${3:-0}" - local package_root="$root/store/v11/links/pkg/1.0.0/hash/node_modules/pkg" + local package_root="$root/node_modules/.pnpm/pkg@1.0.0/node_modules/pkg" mkdir -p "$package_root" mkdir -p "$root/node_modules" cat > "$package_root/package.json" <<'EOF' {"name":"pkg","dependencies":{"dep":"1.0.0"}} EOF - ln -s ../store/v11/links/pkg/1.0.0/hash/node_modules/pkg "$root/node_modules/pkg" + ln -s .pnpm/pkg@1.0.0/node_modules/pkg "$root/node_modules/pkg" if [ "$with_dep" = "1" ]; then mkdir -p "$package_root/node_modules/dep" @@ -88,35 +88,62 @@ EOF ln -s ../source/pkg "$root/node_modules/pkg" } +make_foreign_package_instance_fixture() { + local root="$1" + local sibling_root="$2" + local package_root="$root/node_modules/.pnpm/pkg@1.0.0/node_modules/pkg" + local dependency_root="$sibling_root/node_modules/.pnpm/dep@2.0.0/node_modules/dep" + + mkdir -p "$package_root/node_modules" "$dependency_root" "$root/node_modules" + cat > "$package_root/package.json" <<'EOF' +{"name":"pkg","peerDependencies":{"dep":"^2.0.0"}} +EOF + cat > "$dependency_root/package.json" <<'EOF' +{"name":"dep","version":"2.0.0"} +EOF + ln -s .pnpm/pkg@1.0.0/node_modules/pkg "$root/node_modules/pkg" + ln -s "$dependency_root" "$package_root/node_modules/dep" +} + make_missing_export_fixture() { local root="$1" - local package_root="$root/store/v11/links/pkg/1.0.0/hash/node_modules/pkg" + local package_root="$root/node_modules/.pnpm/pkg@1.0.0/node_modules/pkg" mkdir -p "$package_root" mkdir -p "$root/node_modules" cat > "$package_root/package.json" <<'EOF' {"name":"pkg","files":["src"],"exports":{".":{"default":"./src/index.js"}}} EOF - ln -s ../store/v11/links/pkg/1.0.0/hash/node_modules/pkg "$root/node_modules/pkg" + ln -s .pnpm/pkg@1.0.0/node_modules/pkg "$root/node_modules/pkg" +} + +make_default_files_fixture() { + local root="$1" + local package_root="$root/node_modules/.pnpm/pkg@1.0.0/node_modules/pkg" + + mkdir -p "$package_root/dist" "$root/node_modules" + cat > "$package_root/package.json" <<'EOF' +{"name":"pkg","main":"./dist/index.js"} +EOF + touch "$package_root/dist/index.js" + ln -s .pnpm/pkg@1.0.0/node_modules/pkg "$root/node_modules/pkg" } make_exports_override_stale_main_fixture() { local root="$1" - local package_root="$root/store/v11/links/pkg/1.0.0/hash/node_modules/pkg" + local package_root="$root/node_modules/.pnpm/pkg@1.0.0/node_modules/pkg" - mkdir -p "$package_root/dist" - mkdir -p "$root/node_modules" + mkdir -p "$package_root/dist" "$root/node_modules" cat > "$package_root/package.json" <<'EOF' {"name":"pkg","files":["dist"],"main":"./dist/missing-legacy.cjs","exports":{".":{"import":"./dist/index.js","require":"./dist/index.cjs"}}} EOF - touch "$package_root/dist/index.js" - touch "$package_root/dist/index.cjs" - ln -s ../store/v11/links/pkg/1.0.0/hash/node_modules/pkg "$root/node_modules/pkg" + touch "$package_root/dist/index.js" "$package_root/dist/index.cjs" + ln -s .pnpm/pkg@1.0.0/node_modules/pkg "$root/node_modules/pkg" } make_unshipped_conditional_export_fixture() { local root="$1" - local package_root="$root/store/v11/links/pkg/1.0.0/hash/node_modules/pkg" + local package_root="$root/node_modules/.pnpm/pkg@1.0.0/node_modules/pkg" mkdir -p "$package_root/dist" mkdir -p "$root/node_modules" @@ -124,12 +151,12 @@ make_unshipped_conditional_export_fixture() { {"name":"pkg","files":["dist"],"exports":{".":{"custom-condition":"./src/index.ts","default":"./dist/index.js"}}} EOF touch "$package_root/dist/index.js" - ln -s ../store/v11/links/pkg/1.0.0/hash/node_modules/pkg "$root/node_modules/pkg" + ln -s .pnpm/pkg@1.0.0/node_modules/pkg "$root/node_modules/pkg" } make_missing_conditional_export_alternative_fixture() { local root="$1" - local package_root="$root/store/v11/links/pkg/1.0.0/hash/node_modules/pkg" + local package_root="$root/node_modules/.pnpm/pkg@1.0.0/node_modules/pkg" mkdir -p "$package_root/build" mkdir -p "$root/node_modules" @@ -138,12 +165,12 @@ make_missing_conditional_export_alternative_fixture() { EOF touch "$package_root/build/pkg.esm.js" touch "$package_root/build/pkg.min.js" - ln -s ../store/v11/links/pkg/1.0.0/hash/node_modules/pkg "$root/node_modules/pkg" + ln -s .pnpm/pkg@1.0.0/node_modules/pkg "$root/node_modules/pkg" } make_missing_type_export_fixture() { local root="$1" - local package_root="$root/store/v11/links/pkg/1.0.0/hash/node_modules/pkg" + local package_root="$root/node_modules/.pnpm/pkg@1.0.0/node_modules/pkg" mkdir -p "$package_root/dist" mkdir -p "$root/node_modules" @@ -151,7 +178,7 @@ make_missing_type_export_fixture() { {"name":"pkg","files":["dist"],"exports":{"./internal/module-runner":{"types":"./dist/module-runner.d.ts","default":"./dist/module-runner.js"}}} EOF touch "$package_root/dist/module-runner.js" - ln -s ../store/v11/links/pkg/1.0.0/hash/node_modules/pkg "$root/node_modules/pkg" + ln -s .pnpm/pkg@1.0.0/node_modules/pkg "$root/node_modules/pkg" } make_builtin_dependency_fixture() { @@ -167,7 +194,7 @@ EOF make_extensionless_main_fixture() { local root="$1" - local package_root="$root/store/v11/links/pkg/1.0.0/hash/node_modules/pkg" + local package_root="$root/node_modules/.pnpm/pkg@1.0.0/node_modules/pkg" mkdir -p "$package_root/lib" mkdir -p "$root/node_modules" @@ -175,12 +202,12 @@ make_extensionless_main_fixture() { {"name":"pkg","files":["lib"],"main":"./lib/index","exports":{".":{"default":"./lib/index"}}} EOF touch "$package_root/lib/index.js" - ln -s ../store/v11/links/pkg/1.0.0/hash/node_modules/pkg "$root/node_modules/pkg" + ln -s .pnpm/pkg@1.0.0/node_modules/pkg "$root/node_modules/pkg" } make_missing_subpath_export_fixture() { local root="$1" - local package_root="$root/store/v11/links/pkg/1.0.0/hash/node_modules/pkg" + local package_root="$root/node_modules/.pnpm/pkg@1.0.0/node_modules/pkg" mkdir -p "$package_root/dist" mkdir -p "$root/node_modules" @@ -188,7 +215,7 @@ make_missing_subpath_export_fixture() { {"name":"pkg","files":["dist"],"main":"./dist/index.js","exports":{".":{"default":"./dist/index.js"},"./optional":{"default":"./dist/optional.js"}}} EOF touch "$package_root/dist/index.js" - ln -s ../store/v11/links/pkg/1.0.0/hash/node_modules/pkg "$root/node_modules/pkg" + ln -s .pnpm/pkg@1.0.0/node_modules/pkg "$root/node_modules/pkg" } make_bin_fixture() { @@ -230,59 +257,6 @@ echo "" test_dir="$(mktemp -d)" trap 'rm -rf "$test_dir"' EXIT -echo "Test 1: explicit store-dir takes precedence for GVS links path" -mkdir -p "$test_dir/pnpm-store/v11" "$test_dir/pnpm-home/store/v11" "$test_dir/xdg/pnpm/store/v11" "$test_dir/home/.local/share/pnpm/store/v11" -( - export HOME="$test_dir/home" - export npm_config_store_dir="$test_dir/pnpm-store" - export PNPM_STORE_DIR="$test_dir/ignored-pnpm-store" - export PNPM_HOME="$test_dir/pnpm-home" - export XDG_DATA_HOME="$test_dir/xdg" - assert_eq \ - "$test_dir/pnpm-store/v11/links" \ - "$(resolve_gvs_links_dir)" \ - "resolve_gvs_links_dir prefers npm_config_store_dir" -) - -echo "Test 2: PNPM_STORE_DIR is used when npm_config_store_dir is unset" -( - export HOME="$test_dir/home" - unset npm_config_store_dir - export PNPM_STORE_DIR="$test_dir/pnpm-store" - export PNPM_HOME="$test_dir/pnpm-home" - export XDG_DATA_HOME="$test_dir/xdg" - assert_eq \ - "$test_dir/pnpm-store/v11/links" \ - "$(resolve_gvs_links_dir)" \ - "resolve_gvs_links_dir uses PNPM_STORE_DIR" -) - -echo "Test 3: PNPM_HOME is used when store-dir is unset" -( - export HOME="$test_dir/home" - unset npm_config_store_dir - unset PNPM_STORE_DIR - export PNPM_HOME="$test_dir/pnpm-home" - export XDG_DATA_HOME="$test_dir/xdg" - assert_eq \ - "$test_dir/pnpm-home/store/v11/links" \ - "$(resolve_gvs_links_dir)" \ - "resolve_gvs_links_dir falls back to PNPM_HOME" -) - -echo "Test 4: XDG_DATA_HOME is used when PNPM_HOME is unset" -( - export HOME="$test_dir/home" - unset npm_config_store_dir - unset PNPM_STORE_DIR - unset PNPM_HOME - export XDG_DATA_HOME="$test_dir/xdg" - assert_eq \ - "$test_dir/xdg/pnpm/store/v11/links" \ - "$(resolve_gvs_links_dir)" \ - "resolve_gvs_links_dir uses XDG_DATA_HOME" -) - echo "Test 5: ensure_local_pnpm_home_default sets a workspace-local default" ( unset PNPM_HOME @@ -303,49 +277,32 @@ echo "Test 6: ensure_local_pnpm_home_default preserves an explicit PNPM_HOME" "ensure_local_pnpm_home_default keeps explicit PNPM_HOME" ) -echo "Test 7: Cache fingerprint changes when GVS path changes" -fingerprint_a="$(cache_fingerprint "workspace-hash" "/tmp/a/store/v11/links")" -fingerprint_b="$(cache_fingerprint "workspace-hash" "/tmp/b/store/v11/links")" -if [ "$fingerprint_a" = "$fingerprint_b" ]; then - echo "FAIL: cache fingerprint should change when GVS path changes" - exit 1 -fi - -echo "Test 8: resolve_pnpm_install_contract_file walks up to the repo contract" -contract_fixture="$test_dir/contract-fixture" -mkdir -p "$contract_fixture/packages/app" -printf '{"schemaVersion":1}\n' > "$contract_fixture/pnpm-install-contract.json" -assert_eq \ - "$contract_fixture/pnpm-install-contract.json" \ - "$(resolve_pnpm_install_contract_file "$contract_fixture/packages/app")" \ - "resolve_pnpm_install_contract_file finds ancestor contract" - echo "Test 9: pnpm contract section hashing is stable across JSON key order" contract_a="$test_dir/contract-a.json" contract_b="$test_dir/contract-b.json" cat > "$contract_a" <<'EOF' -{"schemaVersion":1,"gvsLinkContract":{"packageExtensions":{"storybook":{"dependencies":{"@storybook/react-vite":"10.4.6"}}},"allowBuilds":{"esbuild":false}}} +{"schemaVersion":1,"dependencyGraphContract":{"packageExtensions":{"storybook":{"dependencies":{"@storybook/react-vite":"10.4.6"}}},"allowBuilds":{"esbuild":false}}} EOF cat > "$contract_b" <<'EOF' -{"gvsLinkContract":{"allowBuilds":{"esbuild":false},"packageExtensions":{"storybook":{"dependencies":{"@storybook/react-vite":"10.4.6"}}}},"schemaVersion":1} +{"dependencyGraphContract":{"allowBuilds":{"esbuild":false},"packageExtensions":{"storybook":{"dependencies":{"@storybook/react-vite":"10.4.6"}}}},"schemaVersion":1} EOF assert_eq \ - "$(compute_pnpm_contract_section_hash node "$contract_a" gvsLinkContract)" \ - "$(compute_pnpm_contract_section_hash node "$contract_b" gvsLinkContract)" \ + "$(compute_pnpm_contract_section_hash node "$contract_a" dependencyGraphContract)" \ + "$(compute_pnpm_contract_section_hash node "$contract_b" dependencyGraphContract)" \ "contract section hash ignores JSON object key order" -echo "Test 10: policy-only contract changes do not classify as GVS link drift" +echo "Test 10: policy-only contract changes classify as policy drift" contract_policy_old="$test_dir/contract-policy-old.json" contract_policy_new="$test_dir/contract-policy-new.json" cat > "$contract_policy_old" <<'EOF' { "schemaVersion": 1, "packageManager": {"name": "pnpm", "version": "11.8.0"}, - "gvsLinkContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {}}, - "installPolicy": {"enableGlobalVirtualStore": true}, + "dependencyGraphContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {}}, + "installPolicy": {"verifyStoreIntegrity": true}, "storeContract": {"storeDir": ".devenv/pnpm-store-pure-v1"}, "workspaceManifestContract": {"packages": ["packages/app"]}, - "nixIntegration": {"fixedOutputDependencyPrepUsesLiveGlobalVirtualStore": false}, + "nixIntegration": {"liveVirtualStoreScope": "materialization-root"}, "buck2Integration": {"consumeContractArtifact": true} } EOF @@ -353,37 +310,37 @@ cat > "$contract_policy_new" <<'EOF' { "schemaVersion": 1, "packageManager": {"name": "pnpm", "version": "11.8.0"}, - "gvsLinkContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {}}, - "installPolicy": {"enableGlobalVirtualStore": false}, + "dependencyGraphContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {}}, + "installPolicy": {"verifyStoreIntegrity": false}, "storeContract": {"storeDir": ".devenv/pnpm-store-pure-v1"}, "workspaceManifestContract": {"packages": ["packages/app"]}, - "nixIntegration": {"fixedOutputDependencyPrepUsesLiveGlobalVirtualStore": false}, + "nixIntegration": {"liveVirtualStoreScope": "materialization-root"}, "buck2Integration": {"consumeContractArtifact": true} } EOF assert_eq \ "policy" \ "$(classify_pnpm_contract_change node "$contract_policy_old" "$contract_policy_new")" \ - "policy-only contract changes are not gvs-link changes" + "policy-only contract changes classify as policy" -echo "Test 11: packageExtensions changes classify as GVS link drift" -contract_gvs_new="$test_dir/contract-gvs-new.json" -cat > "$contract_gvs_new" <<'EOF' +echo "Test 11: packageExtensions changes classify as dependency-graph drift" +contract_graph_new="$test_dir/contract-graph-new.json" +cat > "$contract_graph_new" <<'EOF' { "schemaVersion": 1, "packageManager": {"name": "pnpm", "version": "11.8.0"}, - "gvsLinkContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {"storybook": {"dependencies": {"@storybook/react-vite": "10.4.6"}}}}, - "installPolicy": {"enableGlobalVirtualStore": true}, + "dependencyGraphContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {"storybook": {"dependencies": {"@storybook/react-vite": "10.4.6"}}}}, + "installPolicy": {"verifyStoreIntegrity": true}, "storeContract": {"storeDir": ".devenv/pnpm-store-pure-v1"}, "workspaceManifestContract": {"packages": ["packages/app"]}, - "nixIntegration": {"fixedOutputDependencyPrepUsesLiveGlobalVirtualStore": false}, + "nixIntegration": {"liveVirtualStoreScope": "materialization-root"}, "buck2Integration": {"consumeContractArtifact": true} } EOF assert_eq \ - "gvs-link" \ - "$(classify_pnpm_contract_change node "$contract_policy_old" "$contract_gvs_new")" \ - "packageExtensions changes are gvs-link changes" + "dependency_graph" \ + "$(classify_pnpm_contract_change node "$contract_policy_old" "$contract_graph_new")" \ + "packageExtensions changes are dependency-graph changes" echo "Test 12: unchanged classified sections report an unknown miss reason" contract_unknown_new="$test_dir/contract-unknown-new.json" @@ -391,11 +348,11 @@ cat > "$contract_unknown_new" <<'EOF' { "schemaVersion": 2, "packageManager": {"name": "pnpm", "version": "11.8.0"}, - "gvsLinkContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {}}, - "installPolicy": {"enableGlobalVirtualStore": true}, + "dependencyGraphContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {}}, + "installPolicy": {"verifyStoreIntegrity": true}, "storeContract": {"storeDir": ".devenv/pnpm-store-pure-v1"}, "workspaceManifestContract": {"packages": ["packages/app"]}, - "nixIntegration": {"fixedOutputDependencyPrepUsesLiveGlobalVirtualStore": true}, + "nixIntegration": {"liveVirtualStoreScope": "materialization-root", "changedMetadata": true}, "buck2Integration": {"consumeContractArtifact": false} } EOF @@ -410,11 +367,11 @@ cat > "$contract_toolchain_new" <<'EOF' { "schemaVersion": 1, "packageManager": {"name": "pnpm", "version": "11.9.0"}, - "gvsLinkContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {}}, - "installPolicy": {"enableGlobalVirtualStore": true}, + "dependencyGraphContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {}}, + "installPolicy": {"verifyStoreIntegrity": true}, "storeContract": {"storeDir": ".devenv/pnpm-store-pure-v1"}, "workspaceManifestContract": {"packages": ["packages/app"]}, - "nixIntegration": {"fixedOutputDependencyPrepUsesLiveGlobalVirtualStore": false}, + "nixIntegration": {"liveVirtualStoreScope": "materialization-root"}, "buck2Integration": {"consumeContractArtifact": true} } EOF @@ -429,11 +386,11 @@ cat > "$contract_store_new" <<'EOF' { "schemaVersion": 1, "packageManager": {"name": "pnpm", "version": "11.8.0"}, - "gvsLinkContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {}}, - "installPolicy": {"enableGlobalVirtualStore": true}, + "dependencyGraphContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {}}, + "installPolicy": {"verifyStoreIntegrity": true}, "storeContract": {"storeDir": ".devenv/pnpm-store-pure-v2"}, "workspaceManifestContract": {"packages": ["packages/app"]}, - "nixIntegration": {"fixedOutputDependencyPrepUsesLiveGlobalVirtualStore": false}, + "nixIntegration": {"liveVirtualStoreScope": "materialization-root"}, "buck2Integration": {"consumeContractArtifact": true} } EOF @@ -448,11 +405,11 @@ cat > "$contract_manifest_new" <<'EOF' { "schemaVersion": 1, "packageManager": {"name": "pnpm", "version": "11.8.0"}, - "gvsLinkContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {}}, - "installPolicy": {"enableGlobalVirtualStore": true}, + "dependencyGraphContract": {"allowBuilds": {"esbuild": false}, "packageExtensions": {}}, + "installPolicy": {"verifyStoreIntegrity": true}, "storeContract": {"storeDir": ".devenv/pnpm-store-pure-v1"}, "workspaceManifestContract": {"packages": ["packages/app", "packages/lib"]}, - "nixIntegration": {"fixedOutputDependencyPrepUsesLiveGlobalVirtualStore": false}, + "nixIntegration": {"liveVirtualStoreScope": "materialization-root"}, "buck2Integration": {"consumeContractArtifact": true} } EOF @@ -467,155 +424,11 @@ cat > "$contract_missing" <<'EOF' {"schemaVersion":1} EOF set +e -compute_pnpm_contract_section_hash node "$contract_missing" gvsLinkContract >/dev/null 2>&1 +compute_pnpm_contract_section_hash node "$contract_missing" dependencyGraphContract >/dev/null 2>&1 exit_code=$? set -e assert_exit_code 1 "$exit_code" "missing section hash should fail" -echo "Test 16a: dependency materialization profile emission is stable and trait-aware" -contract_profile="$test_dir/contract-profile.json" -profile_output="$test_dir/profile.json" -cat > "$contract_profile" <<'EOF' -{ - "schemaVersion": 1, - "packageManager": {"name": "pnpm", "version": "11.8.0"}, - "gvsLinkContract": {"allowBuilds": {}, "packageExtensions": {}, "packageManager": {"name": "pnpm", "version": "11.8.0"}}, - "installPolicy": {"ignoreScripts": true, "verifyStoreIntegrity": true}, - "storeContract": {"owner": "pnpm", "layoutVersion": "v11", "storeDir": ".devenv/pnpm-store-pure-v1"}, - "workspaceManifestContract": {"packages": ["packages/app"]}, - "dependencyMaterializationProfile": { - "schema": "dependency-materialization-profile/v0", - "identityInputs": ["packageManager", "gvsLinkContract", "installPolicy", "storeContract", "workspaceManifestContract"], - "supportedTraits": { - "darwinSplitCas": { - "mutableState": "profile-local", - "sharedContent": "store/v11/files", - "gcAuthority": "shared-pool-coordinator", - "repairAuthority": "devenv" - }, - "isolated": { - "mutableState": "profile-local", - "gcAuthority": "profile-local", - "repairAuthority": "devenv" - } - }, - "nativeBuildPolicyInputs": { - "allowBuilds": "gvsLinkContract.allowBuilds", - "compilerEnv": ["CC", "CXX"] - } - } -} -EOF -emit_dependency_materialization_profile node "$contract_profile" darwinSplitCas "$profile_output" -node - "$profile_output" <<'EOF' -const fs = require('node:fs') -const profile = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')) -if (profile.schema !== 'dependency-materialization-profile/v0') throw new Error('schema drift') -if (!profile.profileId.startsWith('pnpm:')) throw new Error('missing pnpm profile id prefix') -if (profile.store.trait !== 'darwinSplitCas') throw new Error('wrong store trait') -if (profile.authorities.gc !== 'shared-pool-coordinator') throw new Error('wrong gc authority') -if (profile.authorities.repair !== 'devenv') throw new Error('wrong repair authority') -if (!profile.policy.nativeBuildPolicyInputs.compilerEnv.includes('CXX')) throw new Error('missing native compiler policy') -if (profile.evidence.sectionDigests.storeContract.length !== 64) throw new Error('missing section digest') -if (profile.evidence.contractPath.startsWith('/')) throw new Error('contract path should be relative') -EOF -pnpm_contract_supports_dependency_materialization_profile node "$contract_profile" -set +e -pnpm_contract_supports_dependency_materialization_profile node "$contract_missing" >/dev/null 2>&1 -exit_code=$? -set -e -assert_exit_code 1 "$exit_code" "old contract should not require dependency materialization evidence" -set +e -emit_dependency_materialization_profile node "$contract_profile" unknownTrait >/dev/null 2>&1 -exit_code=$? -set -e -assert_exit_code 1 "$exit_code" "unsupported profile trait should fail" - -echo "Test 16b: dependency materialization doctor refuses shared pools and plans coordinated repair" -doctor_root="$test_dir/profile-doctor" -mkdir -p "$doctor_root/profile-a-store/v11" "$doctor_root/profile-b-store/v11" "$doctor_root/shared-files/v11" "$doctor_root/isolated-store/v11/files" -ln -s "$doctor_root/shared-files/v11" "$doctor_root/profile-a-store/v11/files" -ln -s "$doctor_root/shared-files/v11" "$doctor_root/profile-b-store/v11/files" -registry_file="$doctor_root/registry.json" -cat > "$registry_file" < value.schema" \ - "registry schema" -assert_json_field \ - "$registry_profile_id" \ - "$live_registry" \ - "value => value.profiles[0].profileId" \ - "registry profile id" -assert_json_field \ - "refuse-raw-prune" \ - <(dependency_materialization_store_doctor node "$live_registry" "$registry_profile_id") \ - "value => value.decision" \ - "registry shared pool doctor decision" -second_registry="$registry_root/second-registry.json" -write_dependency_materialization_registry node "$profile_output" "$registry_root/second-workspace" "$registry_root/second-store" "$second_registry" "$shared_registry" -assert_json_field \ - "2" \ - "$second_registry" \ - "value => value.profiles.length" \ - "shared registry aggregates sibling roots with the same dependency profile" -assert_eq \ - "2" \ - "$(dependency_materialization_repair_roots node "$second_registry" "$(dependency_materialization_profile_files_pool_id node "$second_registry" "$registry_profile_id")" | wc -l | tr -d ' ')" \ - "repair roots include every workspace sharing the files pool" -discovered_profile_store_dir="$(dependency_materialization_profile_store_dir node "$second_registry" "$registry_profile_id")" -case "$discovered_profile_store_dir" in - "$registry_root/store" | "$registry_root/second-store") ;; - *) - echo "FAIL: profile store dir is discoverable for shared registry refresh" - echo " actual: $discovered_profile_store_dir" - exit 1 - ;; -esac -stale_local_registry="$registry_root/stale-local-registry.json" -write_dependency_materialization_registry node "$profile_output" "$registry_root/workspace" "$registry_root/store" "$stale_local_registry" -assert_eq \ - "2" \ - "$(dependency_materialization_repair_roots node "$shared_registry" "$(dependency_materialization_profile_files_pool_id node "$stale_local_registry" "$registry_profile_id")" | wc -l | tr -d ' ')" \ - "shared registry carries sibling roots missing from stale local registry" -changed_profile="$registry_root/changed-profile.json" -node -e 'const fs=require("node:fs"); const profile=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); profile.profileId += ":changed"; fs.writeFileSync(process.argv[2], JSON.stringify(profile, null, 2) + "\n")' "$profile_output" "$changed_profile" -write_dependency_materialization_registry node "$changed_profile" "$registry_root/workspace" "$registry_root/store" "$live_registry" "$shared_registry" -assert_json_field \ - "2" \ - "$shared_registry" \ - "value => value.profiles.length" \ - "changed profile replaces existing root row instead of adding duplicate sibling" - echo "Test 17: resolve_package_bin prefers package-local .bin shims" bin_fixture="$test_dir/bin-fixture" make_bin_fixture "$bin_fixture" @@ -652,6 +465,17 @@ exit_code=$? set -e assert_exit_code 0 "$exit_code" "projection health passes" +echo "Test 20b: Projection health canonicalizes an aliased materialization root" +healthy_real_dir="$test_dir/healthy-real" +healthy_alias_dir="$test_dir/healthy-alias" +make_projection_fixture "$healthy_real_dir" 1 +ln -s "$healthy_real_dir" "$healthy_alias_dir" +set +e +check_node_modules_links_healthy node "$PROJECTION_SCRIPT" "$healthy_alias_dir/node_modules" +exit_code=$? +set -e +assert_exit_code 0 "$exit_code" "projection health accepts canonical paths through a root alias" + echo "Test 21: Projection health ignores packages that do not export ./package.json" exports_dir="$test_dir/exports" make_projection_fixture "$exports_dir" 1 1 @@ -679,6 +503,16 @@ exit_code=$? set -e assert_exit_code 0 "$exit_code" "projection health skips source link dependency resolution" +echo "Test 23b: Projection health rejects a dependency edge to a foreign sibling pnpm package instance" +foreign_root_dir="$test_dir/foreign-root" +foreign_sibling_dir="$test_dir/foreign-sibling" +make_foreign_package_instance_fixture "$foreign_root_dir" "$foreign_sibling_dir" +set +e +check_node_modules_links_healthy node "$PROJECTION_SCRIPT" "$foreign_root_dir/node_modules" >/dev/null 2>&1 +exit_code=$? +set -e +assert_exit_code 1 "$exit_code" "projection health rejects foreign sibling package instance" + echo "Test 24: Broken node_modules symlink is rejected before projection checks" broken_dir="$test_dir/broken" mkdir -p "$broken_dir/node_modules" @@ -765,181 +599,185 @@ assert_eq \ "$(cd "$exports_override_main_dir" && node -e 'const path = require("node:path"); process.stdout.write(path.basename(require.resolve("pkg")))')" \ "Node resolves the exported require target instead of legacy main" -echo "Test 33: dependency materialization profile is stable for identical contracts" -profile_contract="$test_dir/profile-contract.json" -cat > "$profile_contract" <<'EOF' -{ - "schemaVersion": 1, - "packageManager": {"name": "pnpm", "version": "11.8.0"}, - "storeContract": { - "owner": "pnpm", - "layoutVersion": "v11", - "storeDir": ".devenv/pnpm-store-pure-v1", - "sharedFilesStore": {"enabledForLocalDev": true, "disabledInCi": true}, - "globalVirtualStore": {"enabled": true} - }, - "gvsLinkContract": { - "packageManager": {"name": "pnpm", "version": "11.8.0"}, - "allowBuilds": {"esbuild": false}, - "packageExtensions": {} - }, - "installPolicy": { - "ignoreScripts": true, - "packageImportMethod": "clone-or-copy", - "verifyStoreIntegrity": true - }, - "workspaceManifestContract": { - "injectWorkspacePackages": true, - "packages": ["packages/app", "packages/lib"], - "patchedDependencies": {} - }, - "dependencyMaterializationProfile": { - "schema": "dependency-materialization-profile/v0", - "identityInputs": [ - "packageManager", - "gvsLinkContract", - "installPolicy", - "storeContract", - "workspaceManifestContract" - ], - "supportedTraits": { - "darwinSplitCas": { - "mutableState": "profile-local", - "sharedContent": "store/v11/files", - "gcAuthority": "shared-pool-coordinator", - "repairAuthority": "devenv" - }, - "isolated": { - "mutableState": "profile-local", - "gcAuthority": "profile-local", - "repairAuthority": "devenv" - } - }, - "nativeBuildPolicyInputs": { - "allowBuilds": "gvsLinkContract.allowBuilds", - "compilerEnv": ["CC", "CXX"] - } - } -} -EOF -profile_a="$test_dir/profile-a.json" -profile_b="$test_dir/profile-b.json" -emit_dependency_materialization_profile node "$profile_contract" darwinSplitCas "$profile_a" -emit_dependency_materialization_profile node "$profile_contract" darwinSplitCas "$profile_b" -assert_eq \ - "$(compute_hash < "$profile_a")" \ - "$(compute_hash < "$profile_b")" \ - "dependency profile output is stable" -assert_json_field \ - "shared-pool-coordinator" \ - "$profile_a" \ - "(value) => value.authorities.gc" \ - "dependency profile records gc authority" - -echo "Test 34: source-only files are not dependency profile identity inputs" -mkdir -p "$test_dir/profile-source/packages/app/src" -cp "$profile_contract" "$test_dir/profile-source/pnpm-install-contract.json" -echo "export const value = 1" > "$test_dir/profile-source/packages/app/src/index.ts" +echo "Test 33: Projection digest supports npm's default package file set" +default_files_dir="$test_dir/default-files" +make_default_files_fixture "$default_files_dir" +set +e +NODE_MODULES_HELPER_MODE=projection-hash \ + NODE_MODULES_DIRS="$default_files_dir/node_modules" \ + PNPM_ROOT_MODULES_YAML="$default_files_dir/node_modules/.modules.yaml" \ + node "$PROJECTION_SCRIPT" >/dev/null 2>&1 +exit_code=$? +set -e +assert_exit_code 0 "$exit_code" "projection digest handles packages without a files field" + +echo "Test 34: Linux shared storage selects one full store and automatic zero-copy imports" ( - cd "$test_dir/profile-source" - emit_dependency_materialization_profile node pnpm-install-contract.json darwinSplitCas profile-before.json - echo "export const value = 2" > packages/app/src/index.ts - emit_dependency_materialization_profile node pnpm-install-contract.json darwinSplitCas profile-after.json - assert_json_field \ - "$(node -e "const fs = require('node:fs'); process.stdout.write(JSON.parse(fs.readFileSync('profile-before.json','utf8')).profileId)")" \ - profile-after.json \ - "(value) => value.profileId" \ - "source-only mutations do not affect dependency profile identity" + storage_root="$test_dir/storage-root" + shared_store="$test_dir/shared-store" + mkdir -p "$storage_root" + unset CI PNPM_STORE_DIR PNPM_CONFIG_STORE_DIR npm_config_store_dir + export PNPM_SHARED_STORE_DIR="$shared_store" + export PNPM_MIN_FREE_KIB=0 + configure_pnpm_storage node "$storage_root" "$test_dir/job-store" true + assert_eq "$shared_store" "$npm_config_store_dir" "local roots select the host-owned full store" + assert_eq "auto" "$PNPM_PACKAGE_IMPORT_METHOD" "Linux delegates safe zero-copy selection to pnpm" + test -d "$shared_store/v11/files" + test ! -L "$shared_store/v11/files" ) -echo "Test 35: manifest contract changes dependency profile identity" -manifest_contract="$test_dir/profile-contract-manifest-change.json" -node - "$profile_contract" "$manifest_contract" <<'EOF' -const fs = require('node:fs') -const [from, to] = process.argv.slice(2) -const contract = JSON.parse(fs.readFileSync(from, 'utf8')) -contract.workspaceManifestContract.packages.push('packages/new-member') -fs.writeFileSync(to, `${JSON.stringify(contract, null, 2)}\n`) -EOF -profile_manifest="$test_dir/profile-manifest.json" -emit_dependency_materialization_profile node "$manifest_contract" darwinSplitCas "$profile_manifest" -if [ "$(node -e "const fs = require('node:fs'); process.stdout.write(JSON.parse(fs.readFileSync(process.argv[1], 'utf8')).profileId)" "$profile_a")" = "$(node -e "const fs = require('node:fs'); process.stdout.write(JSON.parse(fs.readFileSync(process.argv[1], 'utf8')).profileId)" "$profile_manifest")" ]; then - echo "FAIL: manifest contract changes dependency profile identity" - exit 1 +echo "Test 34b: historical shared-files pools stay outside the managed store" +( + storage_root="$test_dir/fresh-storage-root" + isolated_home="$test_dir/fresh-storage-home" + historical_pool="$isolated_home/.local/share/pnpm/shared-files/v11" + mkdir -p "$storage_root" "$historical_pool" + printf 'historical\n' > "$historical_pool/sentinel" + export HOME="$isolated_home" + unset CI PNPM_SHARED_STORE_DIR PNPM_SHARED_FILES_DIR PNPM_STORE_DIR PNPM_CONFIG_STORE_DIR npm_config_store_dir + configure_pnpm_storage node "$storage_root" "$test_dir/job-store" true + assert_eq "$isolated_home/.local/share/pnpm/store-shared-v1" "$npm_config_store_dir" "default local store uses its fresh namespace" + test -d "$npm_config_store_dir/v11/files" + test ! -L "$npm_config_store_dir/v11/files" + test -f "$historical_pool/sentinel" +) + +echo "Test 34c: a preexisting external files bridge fails closed" +( + storage_root="$test_dir/bridged-storage-root" + isolated_home="$test_dir/bridged-storage-home" + shared_store="$isolated_home/.local/share/pnpm/store-shared-v1" + historical_pool="$isolated_home/.local/share/pnpm/shared-files/v11" + mkdir -p "$storage_root" "$shared_store/v11" "$historical_pool" + ln -s "$historical_pool" "$shared_store/v11/files" + export HOME="$isolated_home" + unset CI PNPM_SHARED_STORE_DIR PNPM_SHARED_FILES_DIR PNPM_STORE_DIR PNPM_CONFIG_STORE_DIR npm_config_store_dir + set +e + output="$(set -e; configure_pnpm_storage node "$storage_root" "$test_dir/job-store" true 2>&1)" + exit_code=$? + set -e + assert_exit_code 1 "$exit_code" "preexisting external Store Cache bridge should fail before pnpm runs" + grep -qF "Refusing external pnpm Store Cache bridge" <<< "$output" + test -L "$shared_store/v11/files" +) + +echo "Test 34d: a preexisting external store-version bridge fails before mutation" +( + storage_root="$test_dir/version-bridged-storage-root" + isolated_home="$test_dir/version-bridged-storage-home" + shared_store="$isolated_home/.local/share/pnpm/store-shared-v1" + external_version="$test_dir/external-store-version" + mkdir -p "$storage_root" "$shared_store" "$external_version" + ln -s "$external_version" "$shared_store/v11" + export HOME="$isolated_home" + unset CI PNPM_SHARED_STORE_DIR PNPM_SHARED_FILES_DIR PNPM_STORE_DIR PNPM_CONFIG_STORE_DIR npm_config_store_dir + set +e + output="$(set -e; configure_pnpm_storage node "$storage_root" "$test_dir/job-store" true 2>&1)" + exit_code=$? + set -e + assert_exit_code 1 "$exit_code" "external Store Cache version bridge should fail before creating files" + grep -qF "Refusing external pnpm Store Cache version bridge" <<< "$output" + test ! -e "$external_version/files" +) + +echo "Test 34e: recognized legacy files bridge migrates in place under the stable store root" +( + isolated_home="$test_dir/migration-home" + shared_store="$isolated_home/.local/share/pnpm/store-shared-v1" + historical_pool="$isolated_home/.local/share/pnpm/shared-files/v11" + mkdir -p "$shared_store/v11/projects" "$historical_pool" + printf 'historical\n' > "$historical_pool/sentinel" + printf 'stale\n' > "$shared_store/v11/index.db" + ln -s "$historical_pool" "$shared_store/v11/files" + acquire_pnpm_store_cache_lease flock exclusive "$shared_store" 10 + lock_inode_before="$(stat -c %i "$shared_store/.effect-utils-pnpm-store-cache-maintenance.lock")" + migrate_legacy_pnpm_store_cache "$shared_store" "$historical_pool" + lock_inode_after="$(stat -c %i "$shared_store/.effect-utils-pnpm-store-cache-maintenance.lock")" + assert_eq "$lock_inode_before" "$lock_inode_after" "migration preserves the maintenance-lock inode" + test -d "$shared_store/v11/files" + test ! -L "$shared_store/v11/files" + test ! -e "$shared_store/v11/index.db" + test ! -e "$shared_store/v11/projects" + test -f "$historical_pool/sentinel" + migrate_legacy_pnpm_store_cache "$shared_store" "$historical_pool" +) + +echo "Test 34f: unknown legacy files bridge remains untouched" +( + shared_store="$test_dir/unknown-migration-store" + expected_pool="$test_dir/expected-migration-pool" + unknown_pool="$test_dir/unknown-migration-pool" + mkdir -p "$shared_store/v11" "$expected_pool" "$unknown_pool" + ln -s "$unknown_pool" "$shared_store/v11/files" + set +e + output="$(migrate_legacy_pnpm_store_cache "$shared_store" "$expected_pool" 2>&1)" + exit_code=$? + set -e + assert_exit_code 1 "$exit_code" "unknown legacy bridge must fail closed" + grep -qF "Refusing unknown legacy Store Cache bridge" <<< "$output" + test -L "$shared_store/v11/files" +) + +echo "Test 35: Linux zero-copy storage fails closed across filesystems" +if [ -d /dev/shm ] && [ "$(stat -c '%d' /dev/shm)" != "$(stat -c '%d' "$test_dir")" ]; then + cross_device_store="$(mktemp -d /dev/shm/effect-utils-pnpm-store.XXXXXX)" + trap 'rm -rf "$test_dir" "$cross_device_store"' EXIT + ( + storage_root="$test_dir/cross-device-root" + mkdir -p "$storage_root" + unset CI PNPM_STORE_DIR PNPM_CONFIG_STORE_DIR npm_config_store_dir + export PNPM_SHARED_STORE_DIR="$cross_device_store" + export PNPM_MIN_FREE_KIB=0 + set +e + output="$(set -e; configure_pnpm_storage node "$storage_root" "$test_dir/job-store" true 2>&1)" + exit_code=$? + set -e + assert_exit_code 1 "$exit_code" "cross-device zero-copy storage should fail before pnpm runs" + grep -qF "Zero-copy pnpm storage requires one filesystem" <<< "$output" + test ! -e "$cross_device_store/v11/index.db" + ) + rm -rf "$cross_device_store" +else + echo "SKIP: no writable second filesystem is available" fi -echo "Test 36: unknown dependency materialization trait fails closed" -set +e -emit_dependency_materialization_profile node "$profile_contract" unknownTrait >/dev/null 2>&1 -exit_code=$? -set -e -assert_exit_code 1 "$exit_code" "unknown store trait should fail" - -echo "Test 37: store doctor refuses raw prune of a shared files pool" -doctor_registry="$test_dir/doctor-registry.json" -shared_files="$test_dir/shared-files/v11" -shared_root="$test_dir/profile-a/store/v11" -mkdir -p "$shared_files" "$shared_root" -ln -s "$shared_files" "$shared_root/files" -cat > "$doctor_registry" < "$doctor_shared" -assert_json_field \ - "refuse-raw-prune" \ - "$doctor_shared" \ - "(value) => value.decision" \ - "shared pool raw prune is refused" -assert_json_field \ - "profile-a,profile-b" \ - "$doctor_shared" \ - "(value) => value.siblings.join(',')" \ - "shared pool doctor reports sibling profiles" - -echo "Test 38: store doctor allows isolated profile-local pool prune" -isolated_files="$test_dir/isolated/store/v11/files" -mkdir -p "$isolated_files" -isolated_registry="$test_dir/isolated-registry.json" -cat > "$isolated_registry" < "$doctor_isolated" -assert_json_field \ - "allow-profile-local-prune" \ - "$doctor_isolated" \ - "(value) => value.decision" \ - "isolated local pool prune is allowed" - -echo "Test 39: repair plan targets every root sharing a files pool" -repair_plan="$test_dir/repair-plan.json" -dependency_materialization_repair_plan node "$doctor_registry" pool-shared > "$repair_plan" -assert_json_field \ - "repair-all-roots" \ - "$repair_plan" \ - "(value) => value.decision" \ - "shared pool repair plans coordinated rebuild" -assert_json_field \ - "profile-a,profile-b" \ - "$repair_plan" \ - "(value) => value.roots.map((root) => root.profile).join(',')" \ - "shared pool repair plan lists all roots" +echo "Test 35b: CI forces its declared job-local store" +( + storage_root="$test_dir/ci-storage-root" + job_store="$test_dir/ci-job-store" + mkdir -p "$storage_root" + export CI=1 + export PNPM_STORE_DIR="$test_dir/runner-shared-store" + export PNPM_CONFIG_STORE_DIR="$test_dir/runner-shared-store" + export npm_config_store_dir="$test_dir/runner-shared-store" + configure_pnpm_storage node "$storage_root" "$job_store" true + assert_eq "$job_store" "$npm_config_store_dir" "CI store authority remains job-local" + assert_eq "auto" "$PNPM_PACKAGE_IMPORT_METHOD" "CI uses the same native import policy" +) + +echo "Test 35c: capacity checks each distinct writable device exactly once" +( + unset CI + export PNPM_MIN_FREE_KIB=0 + capacity_log="$test_dir/capacity-df.log" + store_dir="$test_dir/capacity-store" + root_dir="$test_dir/capacity-root" + mkdir -p "$store_dir" "$root_dir" + df() { + printf '%s\n' "$*" >> "$capacity_log" + command df "$@" + } + assert_pnpm_storage_capacity node "$store_dir" "$root_dir" + assert_eq "1" "$(wc -l < "$capacity_log" | tr -d ' ')" "same-device boundaries are checked once" + + if [ -d /dev/shm ] && [ "$(stat -c '%d' /dev/shm)" != "$(stat -c '%d' "$test_dir")" ]; then + second_device_root="$(mktemp -d /dev/shm/effect-utils-capacity-root.XXXXXX)" + : > "$capacity_log" + assert_pnpm_storage_capacity node "$store_dir" "$second_device_root" + assert_eq "2" "$(wc -l < "$capacity_log" | tr -d ' ')" "distinct devices are both checked" + rm -rf "$second_device_root" + fi +) echo "" echo "All pnpm task helper tests passed" diff --git a/nix/oxc-config-plugin.nix b/nix/oxc-config-plugin.nix index 57ca0d0a8f..ec836bf622 100644 --- a/nix/oxc-config-plugin.nix +++ b/nix/oxc-config-plugin.nix @@ -29,7 +29,7 @@ let pnpm = pinnedPnpm; }; packageDir = "packages/@overeng/oxc-config"; - pnpmDepsHash = "sha256-CPq+C4iehTkmlqTOBVfUQgpOlGxGVMPxWf7ZvySX6K8="; + pnpmDepsHash = "sha256-GFdhKjd/6KLI8N1X7y3eq83Z/VDAsPsJZgc8zzOp7kE="; srcPath = if builtins.isAttrs src && builtins.hasAttr "outPath" src then diff --git a/nix/workspace-tools/lib/dependency-materialization-profile.nix b/nix/workspace-tools/lib/dependency-materialization-profile.nix index 34b843756d..e11538c9cb 100644 --- a/nix/workspace-tools/lib/dependency-materialization-profile.nix +++ b/nix/workspace-tools/lib/dependency-materialization-profile.nix @@ -21,7 +21,7 @@ let lockfileMode = "frozen"; lifecycleScripts = "ignored"; optionalDependencies = "excluded"; - globalVirtualStore = "disabled-in-nix-prep"; + virtualStoreScope = "materialization-root"; liveStoreState = "purged-from-output"; }; in diff --git a/nix/workspace-tools/lib/mk-pnpm-cli.nix b/nix/workspace-tools/lib/mk-pnpm-cli.nix index d961e59bc4..c6f53eb2aa 100644 --- a/nix/workspace-tools/lib/mk-pnpm-cli.nix +++ b/nix/workspace-tools/lib/mk-pnpm-cli.nix @@ -1330,12 +1330,9 @@ let content-addressed (@_), so matching names guarantee identical content — safe to deduplicate unconditionally. - Outside Nix, pnpm's Global Virtual Store (GVS) solves this by sharing a - single physical store across all install roots. Inside the Nix sandbox GVS - is unavailable (no global store) so it is stripped (see - stripPrepLocalPnpmSettings above), - leaving each root with its own isolated .pnpm store. This dedup step is - the sandbox equivalent of what GVS provides at dev time. + Each Nix install root owns an isolated .pnpm topology. This dedup step + shares already-selected immutable package content without creating a + second dependency-edge selector. Without this, bun's bundler treats each physical copy as a distinct module, creating duplicate singletons (TagProto, GenericTag, Context.Tag registries) diff --git a/nix/workspace-tools/lib/mk-pnpm-cli/tests/fixtures/downstream/flake.nix b/nix/workspace-tools/lib/mk-pnpm-cli/tests/fixtures/downstream/flake.nix index ec16421931..c4b687b492 100644 --- a/nix/workspace-tools/lib/mk-pnpm-cli/tests/fixtures/downstream/flake.nix +++ b/nix/workspace-tools/lib/mk-pnpm-cli/tests/fixtures/downstream/flake.nix @@ -237,6 +237,67 @@ fi printf '%s' "$actual" > "$out" ''; + checks.prepared-workspace-injected-locator-identity = + pkgs.runCommand "mk-pnpm-cli-prepared-workspace-injected-locator-identity" + { + nativeBuildInputs = [ pkgs.nodejs ]; + } + '' + fixture="$PWD/fixture" + mkdir -p \ + "$fixture/sources/alpha" \ + "$fixture/sources/bravo" \ + "$fixture/node_modules/.pnpm/same@file+sources+alpha/node_modules/@fixture/same" \ + "$fixture/node_modules/.pnpm/same@file+sources+bravo/node_modules/@fixture/same" \ + "$fixture/node_modules/.pnpm/same@file+sources+unlisted/node_modules/@fixture/same" + + cat > "$fixture/sources/alpha/package.json" <<'JSON' + {"name":"@fixture/same","fixtureIdentity":"source-alpha"} + JSON + cat > "$fixture/sources/bravo/package.json" <<'JSON' + {"name":"@fixture/same","fixtureIdentity":"source-bravo"} + JSON + cat > "$fixture/node_modules/.pnpm/same@file+sources+alpha/node_modules/@fixture/same/package.json" <<'JSON' + {"name":"@fixture/same","fixtureIdentity":"materialized-alpha"} + JSON + cat > "$fixture/node_modules/.pnpm/same@file+sources+bravo/node_modules/@fixture/same/package.json" <<'JSON' + {"name":"@fixture/same","fixtureIdentity":"materialized-bravo"} + JSON + cat > "$fixture/node_modules/.pnpm/same@file+sources+unlisted/node_modules/@fixture/same/package.json" <<'JSON' + {"name":"@fixture/same","fixtureIdentity":"materialized-unlisted"} + JSON + cat > "$fixture/node_modules/.modules.yaml" <<'YAML' + injectedDeps: + sources/alpha: + - node_modules/.pnpm/same@file+sources+alpha/node_modules/@fixture/same + sources/bravo: + - node_modules/.pnpm/same@file+sources+bravo/node_modules/@fixture/same + layoutVersion: 5 + nodeLinker: isolated + YAML + + ( + cd "$fixture" + PREPARED_WORKSPACE_PLACEHOLDER=/__pnpm_prepared_workspace__ \ + node ${pureEvalFixture.passthru.depsBuildsByInstallRoot.root.rewritePreparedWorkspaceScript} + ) + + alpha_target="$fixture/node_modules/.pnpm/same@file+sources+alpha/node_modules/@fixture/same" + bravo_target="$fixture/node_modules/.pnpm/same@file+sources+bravo/node_modules/@fixture/same" + unlisted_target="$fixture/node_modules/.pnpm/same@file+sources+unlisted/node_modules/@fixture/same" + + test -L "$alpha_target" + test "$(readlink -f "$alpha_target")" = "$fixture/sources/alpha" + test -L "$bravo_target" + test "$(readlink -f "$bravo_target")" = "$fixture/sources/bravo" + + # A name/file+ directory scan would incorrectly rewrite this + # same-name package despite pnpm not assigning it a source locator. + test ! -L "$unlisted_target" + grep -q 'materialized-unlisted' "$unlisted_target/package.json" + + touch "$out" + ''; } ); } diff --git a/nix/workspace-tools/lib/mk-pnpm-cli/tests/run.sh b/nix/workspace-tools/lib/mk-pnpm-cli/tests/run.sh index c083aa205b..6d2c9fa9f5 100755 --- a/nix/workspace-tools/lib/mk-pnpm-cli/tests/run.sh +++ b/nix/workspace-tools/lib/mk-pnpm-cli/tests/run.sh @@ -285,6 +285,11 @@ run_downstream_pure_eval_regression() { exit 1 fi + echo "Check: prepared workspace relinks injected packages by pnpm locator identity" + nix build --no-link --no-write-lock-file \ + --override-input effect-utils "path:$WORKSPACE_REAL/repos/effect-utils" \ + "path:$DOWNSTREAM_DIR#checks.$SYSTEM.prepared-workspace-injected-locator-identity" + echo "Check: downstream staged pnpm-workspace.yaml strips live-worktree pnpm settings" local deps_src deps_src="$(nix build --no-link --no-write-lock-file --print-out-paths \ diff --git a/nix/workspace-tools/lib/mk-pnpm-deps.nix b/nix/workspace-tools/lib/mk-pnpm-deps.nix index 2989dffcdb..023935796e 100644 --- a/nix/workspace-tools/lib/mk-pnpm-deps.nix +++ b/nix/workspace-tools/lib/mk-pnpm-deps.nix @@ -159,6 +159,7 @@ let rewritePreparedWorkspaceScript = pkgs.writeText "rewrite-prepared-workspace.cjs" '' const fs = require("fs"); const path = require("path"); + const { execFileSync } = require("child_process"); const workspaceRoot = process.cwd(); const workspacePlaceholder = process.env.PREPARED_WORKSPACE_PLACEHOLDER; @@ -177,95 +178,87 @@ let left.name.localeCompare(right.name) ); - const workspacePackages = new Map(); + const isWithin = (parentPath, childPath) => { + const relativePath = path.relative(parentPath, childPath); + return relativePath === "" || + (!relativePath.startsWith(`..''${path.sep}`) && relativePath !== ".." && !path.isAbsolute(relativePath)); + }; - const collectWorkspacePackages = (dirPath) => { + const relinkInjectedPackages = (dirPath, relinkedTargets = new Map()) => { for (const entry of sortedDirEntries(dirPath)) { if (!entry.isDirectory()) { continue; } - if (entry.name === "node_modules" || entry.name === ".git") { - continue; - } - const entryPath = path.join(dirPath, entry.name); - const packageJsonPath = path.join(entryPath, "package.json"); - if (fs.existsSync(packageJsonPath)) { - const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")); - if (typeof packageJson.name === "string" && !workspacePackages.has(packageJson.name)) { - workspacePackages.set(packageJson.name, entryPath); + if (entry.name === "node_modules") { + const modulesManifestPath = path.join(entryPath, ".modules.yaml"); + if (!fs.existsSync(modulesManifestPath)) { + continue; } - } - collectWorkspacePackages(entryPath); - } - }; - - const packageDirsInNodeModules = (nodeModulesPath) => { - if (!fs.existsSync(nodeModulesPath)) { - return []; - } - - const packageDirs = []; - for (const entry of sortedDirEntries(nodeModulesPath)) { - if (!entry.isDirectory()) { - continue; - } - - if (entry.name.startsWith("@")) { - const scopeDir = path.join(nodeModulesPath, entry.name); - for (const scopedEntry of sortedDirEntries(scopeDir)) { - if (scopedEntry.isDirectory()) { - packageDirs.push(path.join(scopeDir, scopedEntry.name)); + // pnpm records injected `file:` workspace packages in this manifest. + // Both the source-project key and every materialized target are + // relative to the lockfile directory (the parent of node_modules). + // This locator mapping is authoritative; package names are only + // labels and may legitimately collide across staged source roots. + const modulesManifest = JSON.parse(execFileSync( + "${pkgs.yq-go}/bin/yq", + ["--output-format=json", ".", modulesManifestPath], + { encoding: "utf8" } + )); + const injectedDeps = modulesManifest.injectedDeps ?? {}; + const lockfileDir = path.dirname(entryPath); + + for (const [sourceProjectId, targetIds] of Object.entries(injectedDeps)) { + if (!Array.isArray(targetIds)) { + throw new Error(`invalid injectedDeps targets for ''${sourceProjectId}: ''${modulesManifestPath}`); } - } - } else { - packageDirs.push(path.join(nodeModulesPath, entry.name)); - } - } - return packageDirs; - }; + const sourceProjectDir = path.resolve(lockfileDir, sourceProjectId); + if (!isWithin(workspaceRoot, sourceProjectDir)) { + throw new Error(`injected dependency source escaped prepared workspace: ''${sourceProjectDir}`); + } + if (!fs.existsSync(path.join(sourceProjectDir, "package.json"))) { + throw new Error(`injected dependency source is missing package.json: ''${sourceProjectDir}`); + } + if (!isWithin(workspaceRoot, fs.realpathSync(sourceProjectDir))) { + throw new Error(`injected dependency source resolved outside prepared workspace: ''${sourceProjectDir}`); + } - const relinkLocalVirtualPackages = (dirPath) => { - for (const entry of sortedDirEntries(dirPath)) { - if (!entry.isDirectory()) { - continue; - } + for (const targetId of targetIds) { + if (typeof targetId !== "string") { + throw new Error(`invalid injectedDeps target for ''${sourceProjectId}: ''${modulesManifestPath}`); + } - const entryPath = path.join(dirPath, entry.name); - if (entry.name === ".pnpm") { - for (const virtualEntry of sortedDirEntries(entryPath)) { - if (!virtualEntry.isDirectory() || !virtualEntry.name.includes("file+")) { - continue; - } + const packageDir = path.resolve(lockfileDir, targetId); + if (!isWithin(entryPath, packageDir)) { + throw new Error(`injected dependency target escaped prepared node_modules: ''${packageDir}`); + } - const virtualNodeModulesPath = path.join(entryPath, virtualEntry.name, "node_modules"); - for (const packageDir of packageDirsInNodeModules(virtualNodeModulesPath)) { - const packageJsonPath = path.join(packageDir, "package.json"); - if (!fs.existsSync(packageJsonPath)) { - continue; + const previousSource = relinkedTargets.get(packageDir); + if (previousSource && previousSource !== sourceProjectDir) { + throw new Error( + `conflicting injected dependency locators for ''${packageDir}: ''${previousSource} and ''${sourceProjectDir}` + ); } + relinkedTargets.set(packageDir, sourceProjectDir); - const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")); - const workspacePackageDir = workspacePackages.get(packageJson.name); - if (!workspacePackageDir) { - continue; + if (!fs.existsSync(packageDir)) { + throw new Error(`injected dependency target is missing: ''${packageDir}`); } fs.rmSync(packageDir, { recursive: true, force: true }); - fs.symlinkSync(path.relative(path.dirname(packageDir), workspacePackageDir), packageDir, "dir"); + fs.symlinkSync(path.relative(path.dirname(packageDir), sourceProjectDir), packageDir, "dir"); } } } else { - relinkLocalVirtualPackages(entryPath); + relinkInjectedPackages(entryPath, relinkedTargets); } } }; - collectWorkspacePackages(workspaceRoot); - relinkLocalVirtualPackages(workspaceRoot); + relinkInjectedPackages(workspaceRoot); const rewriteBinScripts = (dirPath, visitedRealPaths = new Set()) => { let realDirPath; @@ -481,7 +474,7 @@ in # strategy changes, even if the recursive output hash stays the same. # Self-hosted darwin runners can otherwise keep colliding with stale temp # output paths for earlier artifact layouts while evaluating the same FOD. - pname = "${name}-pnpm-deps-${srcFingerprint}-v17"; + pname = "${name}-pnpm-deps-${srcFingerprint}-v18"; version = "0.0.0"; inherit src sourceRoot; @@ -804,6 +797,10 @@ in outputHashMode = "recursive"; outputHash = pnpmDepsHash; + + passthru = { + inherit rewritePreparedWorkspaceScript; + }; }; # Generate a shell script snippet that restores a prepared workspace tree. diff --git a/nix/workspace-tools/lib/pnpm-install-policy.nix b/nix/workspace-tools/lib/pnpm-install-policy.nix index 12a5a04947..2c0d2179a5 100644 --- a/nix/workspace-tools/lib/pnpm-install-policy.nix +++ b/nix/workspace-tools/lib/pnpm-install-policy.nix @@ -13,16 +13,15 @@ rec { "--config.strict-store-pkg-content-check=true" "--child-concurrency=1" "--network-concurrency=4" - "--config.package-import-method=clone-or-copy" + "--config.enable-global-virtual-store=false" + "--config.virtual-store-dir=node_modules/.pnpm" "--pm-on-fail=ignore" ]; # The fixed-output builder writes policy through .npmrc because pnpm 11 # rejects some workspace-scoped keys via `pnpm config set --global`. The - # prepared tree is restored directly by downstream builds. That makes FOD - # prep the deliberate exception to live GVS: a GVS node_modules tree points at - # /v11/links, but the builder-local store is not part of the prepared - # output. Keep FOD prep isolated while live devenv installs fully use GVS. + # prepared tree is restored directly by downstream builds. Live and prepared + # installs therefore use the same root-local virtual topology. workspacePrepNpmrcLines = packageImportMethod: [ "virtual-store-dir=node_modules/.pnpm" "package-import-method=${packageImportMethod}" @@ -97,10 +96,9 @@ rec { export NODE_OPTIONS="''${NODE_OPTIONS:+$NODE_OPTIONS }--max-old-space-size=1536" ''; - # Accept only the Darwin teardown failures we have observed after pnpm proves - # materialization finished. Exit 137 is SIGKILL; exit 134 is libuv's abort path - # (`uv__io_poll` assertion). The node_modules checks keep genuine failed or - # partial installs on the normal error path. + # Accept only the Darwin teardown abort that has been observed and recovered + # with the complete shared pnpm store. SIGKILL (137) is deliberately not + # normalized: it provides no exact shared-index recovery evidence. darwinCompletedMaterializationCheckShell = { statusVar, @@ -111,5 +109,5 @@ rec { statusRef = "$" + statusVar; logFileRef = "$" + logFileVar; in - ''{ [ "${statusRef}" -eq 137 ] || [ "${statusRef}" -eq 134 ]; } && [ ${isDarwinShell} = "1" ] && grep -qE 'Progress: .* done$' "${logFileRef}" && [ -d node_modules/.pnpm ] && [ -f node_modules/.modules.yaml ]''; + ''[ "${statusRef}" -eq 134 ] && [ ${isDarwinShell} = "1" ] && grep -qE 'Progress: .* done$' "${logFileRef}" && [ -d node_modules/.pnpm ] && [ -f node_modules/.modules.yaml ]''; } diff --git a/packages/@overeng/ci-tools/nix/build.nix b/packages/@overeng/ci-tools/nix/build.nix index f2c7d8ac03..2c815ba2f0 100644 --- a/packages/@overeng/ci-tools/nix/build.nix +++ b/packages/@overeng/ci-tools/nix/build.nix @@ -8,6 +8,7 @@ }: let + mkSharedHash = hash: { inherit hash; }; pnpm = import ../../../../nix/pnpm.nix { inherit pkgs; }; mkPnpmCli = import ../../../../nix/workspace-tools/lib/mk-pnpm-cli.nix { inherit pkgs pnpm; }; unwrapped = mkPnpmCli { @@ -18,9 +19,7 @@ let workspaceRoot = src; # Managed by the repo FOD refresh workflow — do not edit manually. depsBuilds = { - "." = { - hash = "sha256-vFmEcrqjlypECpGd+d2gLCva06rJCNWbhkXmpwa9fvg="; - }; + "." = mkSharedHash "sha256-wgUckvN2hVyO2sNdt3v9Rq0KPeHwkN9IX+lsk6TFg3M="; }; smokeTestArgs = [ "--help" ]; inherit gitRev commitTs dirty; diff --git a/packages/@overeng/ci-tools/src/cli-command.ts b/packages/@overeng/ci-tools/src/cli-command.ts index 8d44105753..989725cbcc 100644 --- a/packages/@overeng/ci-tools/src/cli-command.ts +++ b/packages/@overeng/ci-tools/src/cli-command.ts @@ -1,3 +1,5 @@ +/* oxlint-disable overeng/exports-first -- Public command trees must be assembled after their private leaf commands are initialized. */ + import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' import { dirname } from 'node:path' @@ -25,13 +27,6 @@ const nonEmptyTextOption = (opts: { readonly name: string; readonly description: const optionalTextOption = (opts: { readonly name: string; readonly description: string }) => Options.text(opts.name).pipe(Options.withDescription(opts.description), Options.withDefault('')) -const expectString = (opts: { readonly value: unknown; readonly path: string }) => { - if (typeof opts.value !== 'string' || opts.value.length === 0) { - throw new Error(`${opts.path} must be a non-empty string`) - } - return opts.value -} - const optionalString = (value: string) => (value.length === 0 ? undefined : value) const optionToUndefined = (value: Option.Option) => @@ -51,9 +46,10 @@ const readInputPaths = (inputPathsJson: string) => { if (Array.isArray(inputPaths) === false) { throw new Error('input paths JSON must decode to an array') } - return inputPaths.map((path, index) => - expectString({ value: path, path: `inputPaths[${index}]` }), - ) + return inputPaths.map((path, index) => { + if (typeof path !== 'string') throw new Error(`inputPaths[${index}] must be a string`) + return path + }) } const latestCreatedAtUtc = (opts: { @@ -103,8 +99,9 @@ const collectBundleCommand = Command.make( Effect.sync(() => { const sources = [] for (const path of readInputPaths(inputPathsJson)) { - if (existsSync(path) === false) { + if (path.length === 0 || existsSync(path) === false) { if (allowMissingInput === true) continue + if (path.length === 0) throw new Error('workflow report input path must be non-empty') throw new Error(`workflow report input file does not exist: ${path}`) } sources.push(readFileSync(path, 'utf8')) diff --git a/packages/@overeng/ci-tools/src/workflow-report.e2e.test.ts b/packages/@overeng/ci-tools/src/workflow-report.e2e.test.ts index dba823e4e6..94ac97a9f8 100644 --- a/packages/@overeng/ci-tools/src/workflow-report.e2e.test.ts +++ b/packages/@overeng/ci-tools/src/workflow-report.e2e.test.ts @@ -53,6 +53,28 @@ const sampleRecord: WorkflowReportRecord = { } describe('workflow-report CLI E2E', () => { + it('treats an absent optional producer output as a missing input', () => { + const workspace = mkdtempSync(join(tmpdir(), 'workflow-report-missing-input-e2e-')) + const bundlePath = join(workspace, 'bundle.json') + + runWorkflowReport([ + 'collect-bundle', + '--bundle-id', + 'deploy-preview', + '--input-paths-json', + JSON.stringify(['']), + '--output-path', + bundlePath, + '--allow-missing-input', + ]) + + expect(decodeWorkflowReportBundleJson(readFileSync(bundlePath, 'utf8'))).toMatchObject({ + _tag: 'WorkflowReportBundle', + bundleId: 'deploy-preview', + records: [], + }) + }) + it('collects marked records, renders managed comments, and locates prior comments', () => { const workspace = mkdtempSync(join(tmpdir(), 'workflow-report-e2e-')) const inputPath = join(workspace, 'deploy.log') diff --git a/packages/@overeng/ci-tools/src/workflow-report.test.ts b/packages/@overeng/ci-tools/src/workflow-report.test.ts index a4bc27473d..06890413e5 100644 --- a/packages/@overeng/ci-tools/src/workflow-report.test.ts +++ b/packages/@overeng/ci-tools/src/workflow-report.test.ts @@ -16,6 +16,7 @@ import { workflowReportBundleJsonSchema, workflowReportManagedMarker, workflowReportRecordLineMarker, + type WorkflowReportManagedState, type WorkflowReportRecord, } from './mod.ts' @@ -202,4 +203,42 @@ describe('managed workflow report comments', () => { expect(nextState.recordOrder).toEqual(['web', 'app']) expect(nextState.entries.map((entry) => entry.entryId)).toEqual(['commit-b', 'commit-a']) }) + + it('evicts oldest history to keep the managed comment below the GitHub limit', () => { + const records = Array.from({ length: 5 }, (_, index) => ({ + ...sampleRecord, + id: `deploy-${index}`, + subject: { id: `subject-${index}` }, + summary: 'x'.repeat(500), + })) + const priorState = Array.from({ length: 20 }).reduce( + (state, _, index) => + deriveWorkflowReportManagedState({ + stateId: 'deploy-preview', + priorState: state, + entryId: `commit-${index}`, + entryLabel: `Commit ${index}`, + createdAtUtc: `2026-05-31T15:${String(index).padStart(2, '0')}:00Z`, + records, + }), + deriveWorkflowReportManagedState({ + stateId: 'deploy-preview', + entryId: 'initial', + entryLabel: 'Initial', + createdAtUtc: '2026-05-31T15:00:00Z', + records, + }), + ) + + const body = renderWorkflowReportCommentBody({ + title: 'Deploy Previews', + noRecordsMessage: 'No previews were deployed.', + state: priorState, + }) + const retainedState = extractWorkflowReportManagedState(body, { stateId: 'deploy-preview' }) + + expect(body.length).toBeLessThanOrEqual(60_000) + expect(retainedState?.entries[0]?.entryId).toBe('commit-19') + expect(retainedState?.entries.length).toBeLessThan(priorState.entries.length) + }) }) diff --git a/packages/@overeng/ci-tools/src/workflow-report.ts b/packages/@overeng/ci-tools/src/workflow-report.ts index cf053ae1dd..007f256077 100644 --- a/packages/@overeng/ci-tools/src/workflow-report.ts +++ b/packages/@overeng/ci-tools/src/workflow-report.ts @@ -694,7 +694,7 @@ const renderRecordsTable = (records: readonly WorkflowReportRecord[], timeZone: ), ] -export const renderWorkflowReportCommentBody = (opts: { +const renderWorkflowReportCommentBodyUnbounded = (opts: { readonly title: string readonly noRecordsMessage: string readonly state: WorkflowReportManagedState @@ -742,3 +742,36 @@ export const renderWorkflowReportCommentBody = (opts: { return `${visibleLines.join('\n')}\n\n${renderWorkflowReportManagedState(opts.state)}\n` } + +const githubCommentBodyMaxLength = 65_536 +const workflowReportCommentBodyMaxLength = 60_000 + +export const renderWorkflowReportCommentBody = (opts: { + readonly title: string + readonly noRecordsMessage: string + readonly state: WorkflowReportManagedState + readonly includeHistory?: boolean +}) => { + let retainedEntries = opts.state.entries + + while (retainedEntries.length > 1) { + const body = renderWorkflowReportCommentBodyUnbounded({ + ...opts, + state: { ...opts.state, entries: retainedEntries }, + }) + if (body.length <= workflowReportCommentBodyMaxLength) return body + retainedEntries = retainedEntries.slice(0, -1) + } + + const body = renderWorkflowReportCommentBodyUnbounded({ + ...opts, + state: { ...opts.state, entries: retainedEntries }, + }) + if (body.length > githubCommentBodyMaxLength) { + throw new Error( + `Current workflow report entry exceeds the GitHub comment limit (${body.length} > ${githubCommentBodyMaxLength} characters)`, + ) + } + + return body +} diff --git a/packages/@overeng/effect-schema-form-aria/package.json.genie.ts b/packages/@overeng/effect-schema-form-aria/package.json.genie.ts index 0d5a30bfb9..5b9d8bfbf0 100644 --- a/packages/@overeng/effect-schema-form-aria/package.json.genie.ts +++ b/packages/@overeng/effect-schema-form-aria/package.json.genie.ts @@ -39,9 +39,6 @@ const runtimeDeps = catalog.compose({ workspace: [schemaFormPkg], external: catalog.pick(...peerDepNames), }, - gvsTypeExtensions: { - 'react-aria-components': catalog.pick('@types/react', '@types/react-dom'), - }, }) export default packageJson( diff --git a/packages/@overeng/genie/nix/build.nix b/packages/@overeng/genie/nix/build.nix index 39e037b9f8..6ce1a3a526 100644 --- a/packages/@overeng/genie/nix/build.nix +++ b/packages/@overeng/genie/nix/build.nix @@ -14,6 +14,7 @@ }: let + mkSharedHash = hash: { inherit hash; }; pnpm = import ../../../../nix/pnpm.nix { inherit pkgs; }; mkPnpmCli = import ../../../../nix/workspace-tools/lib/mk-pnpm-cli.nix { inherit pkgs pnpm; }; opentuiCoreNative = import ../../../../nix/opentui-core-native.nix { inherit pkgs; }; @@ -25,9 +26,7 @@ let workspaceRoot = src; # Managed by the repo FOD refresh workflow — do not edit manually. depsBuilds = { - "." = { - hash = "sha256-/yhNQVzP9CkpjlHP3sKy6uleHKRzl5Ccyr04oKo5SZs="; - }; + "." = mkSharedHash "sha256-8rvIAWJYdWdaORvX634qGaZPNmG31Zc7tvtlTS9O5XI="; }; nativeNodePackages = opentuiCoreNative.packages; inherit gitRev commitTs dirty; diff --git a/packages/@overeng/genie/src/runtime/package-json/catalog.ts b/packages/@overeng/genie/src/runtime/package-json/catalog.ts index 931e010a51..2b40b7227f 100644 --- a/packages/@overeng/genie/src/runtime/package-json/catalog.ts +++ b/packages/@overeng/genie/src/runtime/package-json/catalog.ts @@ -46,12 +46,6 @@ type ComposeArgs< * `install`: also install inherited peer deps of workspace packages using explicit catalog versions. */ mode?: 'manifest' | 'install' - /** GVS: inject @types/* deps into external packages that peer on typed base packages - * but don't ship their own type declarations. - * Keys = external package names, values = catalog.pick(...) of @types/* to inject. - * Aggregated into pnpm-workspace.yaml `packageExtensions` by `rootPnpmWorkspaceYaml`. - * See: pnpm/pnpm#9739 */ - gvsTypeExtensions?: Record } type ComposeResult< @@ -66,7 +60,6 @@ type ComposeResult< devDependencies: TDevDependenciesExternal & WorkspaceDependencyMap peerDependencies: CatalogInput workspace: WorkspaceMetadata - gvsTypeExtensions?: Record } & { readonly [PackageJsonCompositionBrand]: true } @@ -309,7 +302,6 @@ const createComposeFn = devDependencies, peerDependencies, mode = 'manifest', - gvsTypeExtensions, }: ComposeArgs< TDependenciesWorkspace, TDependenciesExternal, @@ -380,7 +372,6 @@ const createComposeFn = ...workspace, deps: [...runtimeWorkspace, ...supportWorkspace, ...peerWorkspace], }, - ...(gvsTypeExtensions !== undefined ? { gvsTypeExtensions } : {}), [PackageJsonCompositionBrand]: true as const, } } diff --git a/packages/@overeng/genie/src/runtime/package-json/mod.ts b/packages/@overeng/genie/src/runtime/package-json/mod.ts index 26748eb607..7413f55a88 100644 --- a/packages/@overeng/genie/src/runtime/package-json/mod.ts +++ b/packages/@overeng/genie/src/runtime/package-json/mod.ts @@ -424,9 +424,6 @@ export type WorkspaceMeta = { export type WorkspacePackageLike = { data: PackageJsonData meta: WorkspaceMeta - /** GVS: per-package `packageExtensions` to inject @types/* into external deps. - * Aggregated into pnpm-workspace.yaml by `rootPnpmWorkspaceYaml`. */ - gvsTypeExtensions?: Record> } /** Package.json genie output that carries workspace-composition metadata. */ @@ -937,11 +934,6 @@ function createPackageJson( return packageJsonValidationMeta })() - const effectiveGvsTypeExtensions = - composition !== undefined && 'gvsTypeExtensions' in composition - ? (composition.gvsTypeExtensions as Record> | undefined) - : undefined - const effectiveWorkspaceMeta = effectiveMeta !== undefined && typeof effectiveMeta === 'object' && @@ -1039,9 +1031,6 @@ function createPackageJson( : []), ], ...(effectiveMeta === undefined ? {} : { meta: effectiveMeta }), - ...(effectiveGvsTypeExtensions === undefined - ? {} - : { gvsTypeExtensions: effectiveGvsTypeExtensions }), }) } diff --git a/packages/@overeng/genie/src/runtime/pnpm-workspace/mod.ts b/packages/@overeng/genie/src/runtime/pnpm-workspace/mod.ts index 26f7ed4ab8..c57af4f02f 100644 --- a/packages/@overeng/genie/src/runtime/pnpm-workspace/mod.ts +++ b/packages/@overeng/genie/src/runtime/pnpm-workspace/mod.ts @@ -737,15 +737,6 @@ export interface PnpmWorkspaceData { */ injectWorkspacePackages?: boolean - /** - * Store dependency contents in a global content-addressed store keyed by - * dependency graph hash. Required for identity convergence: equivalent - * standalone and composed dependency graphs for the same physical source - * tree collapse to one physical instance instead of topology-local duplicates. - * @see https://pnpm.io/settings#enable-global-virtual-store - */ - enableGlobalVirtualStore?: boolean - /** * The location of the content-addressable store. * @see https://pnpm.io/settings#store-dir @@ -1026,10 +1017,6 @@ const buildPnpmWorkspaceYaml = ({ result.injectWorkspacePackages = data.injectWorkspacePackages } - if (data.enableGlobalVirtualStore !== undefined) { - result.enableGlobalVirtualStore = data.enableGlobalVirtualStore - } - if (data.storeDir !== undefined) { result.storeDir = data.storeDir } @@ -1220,38 +1207,9 @@ const rootPnpmWorkspaceYaml = ({ ? projectedMembers : [...new Set([...projectedMembers, ...extraMembers])].toSorted((a, b) => a.localeCompare(b)) - // Aggregate per-package gvsTypeExtensions into workspace-level packageExtensions. - // See: pnpm/pnpm#9739 — GVS stores real paths outside the project tree, breaking - // TypeScript's @types/* resolution. packageExtensions inject @types/* as deps of - // external packages so they appear as siblings in GVS node_modules/. - const aggregatedExtensions: PnpmWorkspaceData['packageExtensions'] = {} - for (const pkg of packages) { - if (pkg.gvsTypeExtensions === undefined) continue - for (const [target, deps] of Object.entries(pkg.gvsTypeExtensions)) { - const existing = aggregatedExtensions[target] - aggregatedExtensions[target] = { - ...existing, - dependencies: { ...existing?.dependencies, ...deps }, - } - } - } - // Merge: user-provided config.packageExtensions overrides aggregated - const mergedExtensions = { ...aggregatedExtensions } - if (config.packageExtensions !== undefined) { - for (const [target, ext] of Object.entries(config.packageExtensions)) { - mergedExtensions[target] = { - ...mergedExtensions[target], - ...ext, - dependencies: { ...mergedExtensions[target]?.dependencies, ...ext.dependencies }, - } - } - } - const hasExtensions = Object.keys(mergedExtensions).length > 0 - const fullConfig = { ...config, packages: allMembers, - ...(hasExtensions === true ? { packageExtensions: mergedExtensions } : {}), } return createGenieOutput({ diff --git a/packages/@overeng/megarepo/nix/build.nix b/packages/@overeng/megarepo/nix/build.nix index 7fb7e89c27..89c38cf6f6 100644 --- a/packages/@overeng/megarepo/nix/build.nix +++ b/packages/@overeng/megarepo/nix/build.nix @@ -12,6 +12,7 @@ }: let + mkSharedHash = hash: { inherit hash; }; pnpm = import ../../../../nix/pnpm.nix { inherit pkgs; }; mkPnpmCli = import ../../../../nix/workspace-tools/lib/mk-pnpm-cli.nix { inherit pkgs pnpm; }; opentuiCoreNative = import ../../../../nix/opentui-core-native.nix { inherit pkgs; }; @@ -23,9 +24,7 @@ let workspaceRoot = src; # Managed by the repo FOD refresh workflow — do not edit manually. depsBuilds = { - "." = { - hash = "sha256-8+b4+ExV+O5vQD88oIpaa89SSQpFMl/HytRP9iaiNaU="; - }; + "." = mkSharedHash "sha256-MiQGQDimBg4ME5mH7kq9s+zwAyzdEzWDtYQn8dbkukM="; }; nativeNodePackages = opentuiCoreNative.packages; smokeTestArgs = [ "--help" ]; diff --git a/packages/@overeng/notion-cli/nix/build.nix b/packages/@overeng/notion-cli/nix/build.nix index 90902458d4..28f0ab8902 100644 --- a/packages/@overeng/notion-cli/nix/build.nix +++ b/packages/@overeng/notion-cli/nix/build.nix @@ -9,6 +9,7 @@ }: let + mkSharedHash = hash: { inherit hash; }; pnpm = import ../../../../nix/pnpm.nix { inherit pkgs; }; mkPnpmCli = import ../../../../nix/workspace-tools/lib/mk-pnpm-cli.nix { inherit pkgs pnpm; }; opentuiCoreNative = import ../../../../nix/opentui-core-native.nix { inherit pkgs; }; @@ -32,9 +33,7 @@ let installRuntimeWorkspace = true; # Managed by the repo FOD refresh workflow — do not edit manually. depsBuilds = { - "." = { - hash = "sha256-EkZopKKjTHR7pQI3BQ89VSDAFCtr/Vl0qiFapwF81/g="; - }; + "." = mkSharedHash "sha256-qijNTviHT0LpS2NBEGsgo0QoWfPVNcPJcr34CfeVM0g="; }; nativeNodePackages = opentuiCoreNative.packages; inherit gitRev commitTs dirty; diff --git a/packages/@overeng/notion-md/nix/build.nix b/packages/@overeng/notion-md/nix/build.nix index e44cd8a28d..1ceffe62f1 100644 --- a/packages/@overeng/notion-md/nix/build.nix +++ b/packages/@overeng/notion-md/nix/build.nix @@ -9,6 +9,7 @@ }: let + mkSharedHash = hash: { inherit hash; }; pnpm = import ../../../../nix/pnpm.nix { inherit pkgs; }; mkPnpmCli = import ../../../../nix/workspace-tools/lib/mk-pnpm-cli.nix { inherit pkgs pnpm; }; unwrapped = mkPnpmCli { @@ -19,9 +20,7 @@ let workspaceRoot = src; # Managed by the repo FOD refresh workflow — do not edit manually. depsBuilds = { - "." = { - hash = "sha256-YPKCaUgQ+d/ADoItdv/1SHSHFPcK/LPIcJBnNS4OOJI="; - }; + "." = mkSharedHash "sha256-K1d0ehf+UsBAhz3s+ABS+rXDxNc+gPCGRKJnYJZRQIc="; }; smokeTestArgs = [ "--help" ]; inherit gitRev commitTs dirty; diff --git a/packages/@overeng/otel-scrape/tests/cli.rs b/packages/@overeng/otel-scrape/tests/cli.rs index db2338b2bf..9fabdcd82e 100644 --- a/packages/@overeng/otel-scrape/tests/cli.rs +++ b/packages/@overeng/otel-scrape/tests/cli.rs @@ -2529,6 +2529,7 @@ fn node_cpuprofile_adapter_writes_resolvable_profile_without_leaking_private_inp let summary = dir.path().join("summary.json"); let cas_root = dir.path().join("cas"); let private_arg = "PRIVATE_ARG_MARKER"; + let private_source = "PRIVATE_SOURCE_MARKER"; let out = otel_scrape() .args(["--adapter", "node-cpuprofile"]) @@ -2541,7 +2542,7 @@ fn node_cpuprofile_adapter_writes_resolvable_profile_without_leaking_private_inp "--", "node", "-e", - "for (let i = 0; i < 100000; i++) Math.sqrt(i); console.log(process.argv[1])", + "/* PRIVATE_SOURCE_MARKER */ for (let i = 0; i < 100000; i++) Math.sqrt(i); console.log(process.argv[1])", private_arg, ]) .output() @@ -2588,9 +2589,9 @@ fn node_cpuprofile_adapter_writes_resolvable_profile_without_leaking_private_inp let summary_json = serde_json::to_string(&summary).unwrap(); let body_json = serde_json::to_string(&body).unwrap(); assert!(!summary_json.contains(private_arg)); - assert!(!summary_json.contains("100000")); + assert!(!summary_json.contains(private_source)); assert!(!body_json.contains(private_arg)); - assert!(!body_json.contains("100000")); + assert!(!body_json.contains(private_source)); let span = &body["resourceSpans"][0]["scopeSpans"][0]["spans"][0]; let events = span["events"].as_array().unwrap(); diff --git a/packages/@overeng/pty-effect/src/client.ts b/packages/@overeng/pty-effect/src/client.ts index 23b4a95255..bfef91fe8f 100644 --- a/packages/@overeng/pty-effect/src/client.ts +++ b/packages/@overeng/pty-effect/src/client.ts @@ -348,7 +348,7 @@ const buildSpawnOpts = (spec: PtyDaemonSpec): SpawnDaemonOptions => { if (spec.tags !== undefined && Object.keys(spec.tags).length > 0) opts.tags = { ...spec.tags } /* When running under Bun, route the detached daemon through Node so the * PTY server can load its `node-pty` native addon. Preserve symlinks so - * pnpm GVS/link projections resolve the same graph in the daemon process. */ + * the composed runtime topology resolves the same graph in the daemon process. */ if (process.versions.bun !== undefined) { opts.launcher = { command: process.env['NODE_BIN'] ?? 'node', diff --git a/packages/@overeng/tui-stories/nix/build.nix b/packages/@overeng/tui-stories/nix/build.nix index 3cbdf3c97b..d61b7cdf8f 100644 --- a/packages/@overeng/tui-stories/nix/build.nix +++ b/packages/@overeng/tui-stories/nix/build.nix @@ -9,6 +9,7 @@ }: let + mkSharedHash = hash: { inherit hash; }; pnpm = import ../../../../nix/pnpm.nix { inherit pkgs; }; mkPnpmCli = import ../../../../nix/workspace-tools/lib/mk-pnpm-cli.nix { inherit pkgs pnpm; }; opentuiCoreNative = import ../../../../nix/opentui-core-native.nix { inherit pkgs; }; @@ -20,9 +21,7 @@ let workspaceRoot = src; # Managed by the repo FOD refresh workflow — do not edit manually. depsBuilds = { - "." = { - hash = "sha256-xY3B/jEzwWMMkhd8oQ8zvtyPE4iOoHh878tI1j+mPIc="; - }; + "." = mkSharedHash "sha256-+nYeFYAEEOSdTxIlvuq9ihKaFRh3wYeS6LN2OAAKCz8="; }; nativeNodePackages = opentuiCoreNative.packages; inherit gitRev commitTs dirty; diff --git a/pnpm-install-contract.json b/pnpm-install-contract.json index 3d65ace723..6c10c6f48b 100644 --- a/pnpm-install-contract.json +++ b/pnpm-install-contract.json @@ -1,47 +1,6 @@ { "contract": "effect-utils/pnpm-install-contract", - "dependencyMaterializationProfile": { - "buck2Boundary": { - "consumesEvidence": true, - "ownsLiveMaterialization": false - }, - "identityInputs": [ - "packageManager", - "gvsLinkContract", - "installPolicy", - "storeContract", - "workspaceManifestContract" - ], - "nativeBuildPolicyInputs": { - "allowBuilds": "gvsLinkContract.allowBuilds", - "compilerEnv": ["CC", "CXX"] - }, - "schema": "dependency-materialization-profile/v0", - "supportedTraits": { - "ciJobLocal": { - "gcAuthority": "profile-local", - "mutableState": "job-local", - "repairAuthority": "ci-job" - }, - "darwinSplitCas": { - "gcAuthority": "shared-pool-coordinator", - "mutableState": "profile-local", - "repairAuthority": "devenv", - "sharedContent": "store/v11/files" - }, - "isolated": { - "gcAuthority": "profile-local", - "mutableState": "profile-local", - "repairAuthority": "devenv" - }, - "nixPreparedDeps": { - "gcAuthority": "nix-store", - "mutableState": "none", - "repairAuthority": "evergreen-fod" - } - } - }, - "gvsLinkContract": { + "dependencyGraphContract": { "allowBuilds": { "@myobie/pty": false, "@parcel/watcher": false, @@ -53,12 +12,6 @@ "unix-dgram": false }, "packageExtensions": { - "react-aria-components": { - "dependencies": { - "@types/react": "19.2.17", - "@types/react-dom": "19.2.3" - } - }, "storybook": { "dependencies": { "@storybook/react-vite": "10.4.6" @@ -75,7 +28,16 @@ "ignoreScripts": true, "minimumReleaseAgeExclude": ["@effect/platform", "@types/node", "effect"], "optimisticRepeatInstall": false, - "packageImportMethod": "clone-or-copy", + "packageImportMethod": { + "live": { + "linuxSameDeviceRequired": true, + "method": "auto", + "owner": "pnpm" + }, + "nixPreparedDependencies": { + "scope": "independent-builder-policy" + } + }, "peerDependencyRules": { "allowedVersions": { "@effect/experimental": ">=0.58.0", @@ -104,31 +66,39 @@ "consumeContractArtifact": true }, "nixIntegration": { - "fixedOutputDependencyPrepUsesLiveGlobalVirtualStore": false, - "liveInstallUsesGlobalVirtualStore": true + "fixedOutputDependencyPrepUsesSameVirtualStoreScope": true, + "liveVirtualStoreScope": "materialization-root" }, "pnpmStoreOwnership": { - "filesLifecycle": "pnpm-owned content-addressed files store", - "linksLifecycle": "pnpm-owned rebuildable dependency-graph projection", - "projectsLifecycle": "pnpm-owned store prune reachability registry" + "cacheLifecycle": "pnpm-owned disposable Store Cache", + "derivedIndexLifecycle": "shared only inside one same-user trust boundary", + "virtualStoreLifecycle": "Materialization-Root-owned rebuildable dependency graph" } }, "packageManager": { "name": "pnpm", "version": "11.8.0" }, - "schemaVersion": 1, + "schemaVersion": 2, "storeContract": { - "globalVirtualStore": { - "enabled": true + "ci": { + "scope": "job" }, "layoutVersion": "v11", - "owner": "pnpm", - "sharedFilesStore": { - "disabledInCi": true, - "enabledForLocalDev": true + "localDevelopment": { + "contentAddressedFiles": "shared", + "defaultPath": "~/.local/share/pnpm/store-shared-v1", + "derivedIndex": "shared-pnpm-owned", + "pathOverrideEnvironmentVariable": "PNPM_SHARED_STORE_DIR", + "scope": "host-user", + "trustBoundary": "same-os-user" }, - "storeDir": ".devenv/pnpm-store-pure-v1" + "owner": "pnpm", + "virtualStore": { + "global": false, + "path": "node_modules/.pnpm", + "scope": "materialization-root" + } }, "workspaceManifestContract": { "allowUnusedPatches": true, diff --git a/pnpm-install-contract.json.genie.ts b/pnpm-install-contract.json.genie.ts index a91a58696f..2aeb2e5ab1 100644 --- a/pnpm-install-contract.json.genie.ts +++ b/pnpm-install-contract.json.genie.ts @@ -1,4 +1,7 @@ -import { projectionArtifact } from './genie/external.ts' +import { + pnpmInstallStorageContractV2 as storage, + projectionArtifact, +} from './genie/external.ts' import rootPackageJson from './package.json.genie.ts' import rootPnpmWorkspaceYaml from './pnpm-workspace.yaml.genie.ts' @@ -7,28 +10,16 @@ const pnpmVersion = packageManager.startsWith('pnpm@') ? packageManager.slice('pnpm@'.length) : packageManager const workspaceData = rootPnpmWorkspaceYaml.data - export default projectionArtifact.json({ - schemaVersion: 1, + schemaVersion: 2, data: { contract: 'effect-utils/pnpm-install-contract', packageManager: { name: 'pnpm', version: pnpmVersion, }, - storeContract: { - owner: 'pnpm', - layoutVersion: 'v11', - storeDir: workspaceData.storeDir, - sharedFilesStore: { - enabledForLocalDev: true, - disabledInCi: true, - }, - globalVirtualStore: { - enabled: workspaceData.enableGlobalVirtualStore, - }, - }, - gvsLinkContract: { + storeContract: storage.storeContract, + dependencyGraphContract: { packageManager: { name: 'pnpm', version: pnpmVersion, @@ -41,7 +32,7 @@ export default projectionArtifact.json({ ignoreScripts: workspaceData.ignoreScripts, minimumReleaseAgeExclude: workspaceData.minimumReleaseAgeExclude, optimisticRepeatInstall: workspaceData.optimisticRepeatInstall, - packageImportMethod: workspaceData.packageImportMethod, + packageImportMethod: storage.packageImportMethod, peerDependencyRules: workspaceData.peerDependencyRules, pmOnFail: workspaceData.pmOnFail, sideEffectsCache: workspaceData.sideEffectsCache, @@ -59,59 +50,18 @@ export default projectionArtifact.json({ }, metadata: { pnpmStoreOwnership: { - filesLifecycle: 'pnpm-owned content-addressed files store', - linksLifecycle: 'pnpm-owned rebuildable dependency-graph projection', - projectsLifecycle: 'pnpm-owned store prune reachability registry', + cacheLifecycle: 'pnpm-owned disposable Store Cache', + derivedIndexLifecycle: 'shared only inside one same-user trust boundary', + virtualStoreLifecycle: 'Materialization-Root-owned rebuildable dependency graph', }, nixIntegration: { - liveInstallUsesGlobalVirtualStore: true, - fixedOutputDependencyPrepUsesLiveGlobalVirtualStore: false, + liveVirtualStoreScope: 'materialization-root', + fixedOutputDependencyPrepUsesSameVirtualStoreScope: true, }, buck2Integration: { consumeContractArtifact: true, avoidNodeModulesLayoutAsApi: true, }, }, - dependencyMaterializationProfile: { - schema: 'dependency-materialization-profile/v0', - identityInputs: [ - 'packageManager', - 'gvsLinkContract', - 'installPolicy', - 'storeContract', - 'workspaceManifestContract', - ], - supportedTraits: { - ciJobLocal: { - mutableState: 'job-local', - gcAuthority: 'profile-local', - repairAuthority: 'ci-job', - }, - darwinSplitCas: { - mutableState: 'profile-local', - sharedContent: 'store/v11/files', - gcAuthority: 'shared-pool-coordinator', - repairAuthority: 'devenv', - }, - isolated: { - mutableState: 'profile-local', - gcAuthority: 'profile-local', - repairAuthority: 'devenv', - }, - nixPreparedDeps: { - mutableState: 'none', - gcAuthority: 'nix-store', - repairAuthority: 'evergreen-fod', - }, - }, - nativeBuildPolicyInputs: { - allowBuilds: 'gvsLinkContract.allowBuilds', - compilerEnv: ['CC', 'CXX'], - }, - buck2Boundary: { - consumesEvidence: true, - ownsLiveMaterialization: false, - }, - }, }, }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 44cd7a788f..f6869e1da1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5,7 +5,7 @@ settings: excludeLinksFromLockfile: false injectWorkspacePackages: true -packageExtensionsChecksum: sha256-gJVbGKCzsiTLmVnQ74k2GtcbKOOuGvVsGVwXhM0lg4Q= +packageExtensionsChecksum: sha256-4HimX0luQTvo3jbLtYHagQKuv0eSu+BxzCFfrE1bdnM= patchedDependencies: '@effect/platform@0.96.2': 08d6466db56675b7a32a3a3c64815a5b784f583b310b6758471a97d3db6edd32 @@ -7772,8 +7772,6 @@ snapshots: '@internationalized/date': 3.12.2 '@react-types/shared': 3.36.0(react@19.2.7) '@swc/helpers': 0.5.21 - '@types/react': 19.2.17 - '@types/react-dom': 19.2.3(@types/react@19.2.17) client-only: 0.0.1 react: 19.2.7 react-aria: 3.50.0(react-dom@19.2.7(react@19.2.7))(react@19.2.7) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index dac964786b..e2cce46cf6 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -45,10 +45,6 @@ patchedDependencies: allowUnusedPatches: true packageExtensions: - react-aria-components: - dependencies: - '@types/react': 19.2.17 - '@types/react-dom': 19.2.3 storybook: dependencies: '@storybook/react-vite': 10.4.6 @@ -85,11 +81,7 @@ strictPeerDependencies: true injectWorkspacePackages: true -enableGlobalVirtualStore: true - -storeDir: .devenv/pnpm-store-pure-v1 - -packageImportMethod: clone-or-copy +packageImportMethod: auto optimisticRepeatInstall: false