feat(protocol): publish Git and Xet state through capsules

.github/workflows/architecture.yml

@@ -175,6 +175,9 @@ jobs:
       CRAB_HTTP_CELL_TEST_BUCKET: crab-cell-runtime
       CRAB_HTTP_CELL_TEST_ENDPOINT: http://127.0.0.1:9000
       CRAB_HTTP_CELL_TEST_PREFIX: server-${{ github.run_id }}-${{ github.run_attempt }}
+      CRAB_CELL_TEST_BUCKET: crab-cell-runtime
+      CRAB_CELL_TEST_ENDPOINT: http://127.0.0.1:9000
+      CRAB_CELL_TEST_PREFIX: public-${{ github.run_id }}-${{ github.run_attempt }}
       RUSTFS_IMAGE: ghcr.io/rustfs/rustfs:1.0.0-glibc@sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858
 
     steps:

.github/workflows/git-protocol-v2-partial-clone.yml

@@ -27,6 +27,7 @@ on:
       - "crab/src/cmd/fsck*.rs"
       - "crab/src/cmd/history_recovery.rs"
       - "crab/src/cmd/metadb.rs"
+      - "crab/src/cmd/push.rs"
       - "crab/src/cmd/repack.rs"
       - "crab/src/cmd/gc/**"
       - "crab/src/replication/**"
@@ -80,6 +81,7 @@ on:
       - "crab/src/cmd/fsck*.rs"
       - "crab/src/cmd/history_recovery.rs"
       - "crab/src/cmd/metadb.rs"
+      - "crab/src/cmd/push.rs"
       - "crab/src/cmd/repack.rs"
       - "crab/src/cmd/gc/**"
       - "crab/src/replication/**"
@@ -185,6 +187,7 @@ jobs:
             git::push::tests::shared_xorb_non_atomic_replan \
             git::push::tests::ref_lease
           cargo test -p crab --lib git::remote_helper::tests::list_session_does_not_open_a_damaged_staging_index --locked --features gix-transport
+          cargo test -p crab --lib git::remote_helper::tests::capsule_promisor_fetch_ --locked --features gix-transport
           cargo test -p crab-metadata --lib file_index_lookup --locked --features file-index-reader
           cargo test -p crab-metadata --lib git_visibility::tests --locked --features remote-index
           cargo test -p crab-metadata --lib ref_journal::tests --locked --features remote-index

.github/workflows/large-repository-rustfs.yml

@@ -6,6 +6,9 @@ on:
       - ".github/workflows/large-repository-rustfs.yml"
       - "crab/scripts/e2e/run_large_repo_rustfs.py"
       - "crab/scripts/e2e/test_verify_large_repo_rustfs_report.py"
+      - "crab/scripts/e2e/run_capsule_k8s_rustfs.py"
+      - "crab/scripts/e2e/test_run_capsule_k8s_rustfs.py"
+      - "crab/docs/design/capsule-layered-packs.md"
       - "crab/scripts/verify-large-repo-rustfs-report.py"
       - "crab/docs/guides/large-repository-qualification.md"
       - "crates/crab-read/src/upload_pack.rs"
@@ -37,6 +40,9 @@ on:
       - ".github/workflows/large-repository-rustfs.yml"
       - "crab/scripts/e2e/run_large_repo_rustfs.py"
       - "crab/scripts/e2e/test_verify_large_repo_rustfs_report.py"
+      - "crab/scripts/e2e/run_capsule_k8s_rustfs.py"
+      - "crab/scripts/e2e/test_run_capsule_k8s_rustfs.py"
+      - "crab/docs/design/capsule-layered-packs.md"
       - "crab/scripts/verify-large-repo-rustfs-report.py"
       - "crab/docs/guides/large-repository-qualification.md"
       - "crates/crab-read/src/upload_pack.rs"
@@ -85,9 +91,11 @@ jobs:
       - name: Verify report contract tests
         run: |
           python3 -m unittest crab/scripts/e2e/test_verify_large_repo_rustfs_report.py
+          python3 -m unittest crab/scripts/e2e/test_run_capsule_k8s_rustfs.py
           python3 -m py_compile \
             crab/scripts/e2e/run_large_repo_rustfs.py \
-            crab/scripts/verify-large-repo-rustfs-report.py
+            crab/scripts/verify-large-repo-rustfs-report.py \
+            crab/scripts/e2e/run_capsule_k8s_rustfs.py
 
   kubernetes-rustfs:
     name: Kubernetes 1,000-commit RustFS qualification

.github/workflows/s3-gateway.yml

@@ -295,9 +295,11 @@ jobs:
       - name: Build minimal Crab fixture publisher
         env:
           CARGO_TARGET_DIR: ${{ runner.temp }}/crab-s3-gateway-publisher-target
-        run: >-
-          cargo build --release --locked -p crab --bin crab
-          --no-default-features --features simd-accel,gix-pathmatch
+        run: |
+          cargo build --release --locked -p crab --bin crab \
+            --no-default-features --features simd-accel,gix-pathmatch
+          cargo build --release --locked -p crab-sdk \
+            --example s3_gateway_fixture --features content,write
 
       - name: Build locked gateway image
         id: build
@@ -485,11 +487,6 @@ jobs:
               for _ in range(64):
                   stream.write(source.randbytes(1024 * 1024))
           PY
-          cp "${source_root}/qualification/xet-large.bin" \
-            "${source_root}/qualification/xet-duplicate.bin"
-          git -C "${source_root}" init -q -b main
-          git -C "${source_root}" config user.name 'S3 Xet qualification'
-          git -C "${source_root}" config user.email 'qualification@example.invalid'
           export AWS_ACCESS_KEY_ID="${RUSTFS_ACCESS_KEY}"
           export AWS_SECRET_ACCESS_KEY="${RUSTFS_SECRET_KEY}"
           export AWS_DEFAULT_REGION=us-east-1
@@ -500,55 +497,28 @@ jobs:
           export AWS_EC2_METADATA_DISABLED=true
           export AWS_VIRTUAL_HOSTED_STYLE_REQUEST=false
           export CRAB_CACHE_DIR="${xet_root}/publisher-cache"
-          crab_bin="${RUNNER_TEMP}/crab-s3-gateway-publisher-target/release/crab"
-          (
-            cd "${source_root}"
-            "${crab_bin}" init --storage-provider s3 \
-              crab://crab-s3-gateway-qualification/repositories/qualification
-            "${crab_bin}" track '*.bin'
-            "${crab_bin}" add qualification/xet-large.bin \
-              qualification/xet-duplicate.bin
-            git show :qualification/xet-large.bin > "${xet_root}/xet-pointer-staged"
-            LISTING_ROOT="${source_root}/qualification/listing" \
-            XET_POINTER="${xet_root}/xet-pointer-staged" python3 - <<'PY'
-          import os
-          from pathlib import Path
-
-          root = Path(os.environ["LISTING_ROOT"])
-          pointer = Path(os.environ["XET_POINTER"]).read_bytes()
-          root.mkdir(parents=True)
-          for index in range(10_000):
-              (root / f"flat-{index:05}.pointer").write_bytes(pointer)
-          for group in ("group-a", "group-b"):
-              directory = root / group
-              directory.mkdir()
-              for index in range(16):
-                  (directory / f"item-{index:02}.pointer").write_bytes(pointer)
-          PY
-            aws s3api create-bucket \
-              --bucket crab-s3-gateway-listing-baseline >/dev/null
-            aws s3 cp qualification/listing \
-              s3://crab-s3-gateway-listing-baseline/main/qualification/listing/ \
-              --recursive --only-show-errors --no-progress
-            git add crab.toml .gitattributes qualification/listing
-            git commit -q -m 'add Xet range qualification fixture'
-            "${crab_bin}" push --upload-concurrency 0 \
-              origin HEAD:refs/heads/main
-            git show HEAD:qualification/xet-large.bin > "${xet_root}/xet-pointer"
-            git show HEAD:qualification/xet-duplicate.bin \
-              > "${xet_root}/xet-duplicate-pointer"
-          )
+          "${RUNNER_TEMP}/crab-s3-gateway-publisher-target/release/examples/s3_gateway_fixture" \
+            crab-s3-gateway-qualification repositories/qualification \
+            "${xet_root}/publisher-cache" \
+            "${source_root}/qualification/xet-large.bin" \
+            "${xet_root}/xet-pointer" \
+            "${source_root}/qualification/listing"
+          cp "${xet_root}/xet-pointer" "${xet_root}/xet-duplicate-pointer"
+          aws s3api create-bucket \
+            --bucket crab-s3-gateway-listing-baseline >/dev/null
+          aws s3 cp "${source_root}/qualification/listing" \
+            s3://crab-s3-gateway-listing-baseline/main/qualification/listing/ \
+            --recursive --only-show-errors --no-progress
           grep -Eq '^version https://crab.build/spec/v1$' \
             "${xet_root}/xet-pointer"
           grep -Fx 'size 67108864' "${xet_root}/xet-pointer"
           test "$(sed -n 's/^file-hash //p' "${xet_root}/xet-pointer")" = \
             "$(sed -n 's/^file-hash //p' "${xet_root}/xet-duplicate-pointer")"
-          git -C "${source_root}" ls-tree -r --name-only HEAD \
-            qualification/listing | wc -l | tr -d ' ' \
+          find "${source_root}/qualification/listing" -type f | wc -l | tr -d ' ' \
             > "${xet_root}/listing-object-count"
           test "$(cat "${xet_root}/listing-object-count")" = 10032
-          git -C "${source_root}" ls-tree -r HEAD qualification/listing | \
-            awk '{print $3}' | sort -u | wc -l | tr -d ' ' \
+          find "${source_root}/qualification/listing" -type f -print0 | \
+            xargs -0 sha256sum | awk '{print $1}' | sort -u | wc -l | tr -d ' ' \
             > "${xet_root}/listing-blob-count"
           test "$(cat "${xet_root}/listing-blob-count")" = 1
           aws --endpoint-url http://127.0.0.1:19000 s3api list-objects-v2 \

CONTEXT.md

@@ -100,3 +100,22 @@ _Avoid_: Cache-warm or sparse-active Cell
 An advisory, revision-pinned ranking or movement decision derived from signed
 fleet capacity. It never grants ownership; Cell authority remains decisive.
 _Avoid_: Ownership record or scheduler assignment
+## Capsule protocol language
+
+**Capsule**:
+A content-addressed immutable object that co-locates one ref transaction with
+its Git pack, authenticated indexes, and external Xorb/Shard dependency evidence.
+Large-file payloads remain outside the capsule.
+_Avoid_: Pack, because a capsule contains a Git pack plus non-pack evidence
+
+**Repository root**:
+The bounded mutable record for checkpoint, symbolic HEAD, GC, and maintenance
+transitions. Ordinary branch publication uses independently conditional per-ref
+heads rather than contending on this record.
+_Avoid_: Manifest, when referring to the capsule-protocol storage protocol
+
+**Checkpoint**:
+An immutable repository view binding refs, indexes, and a layered pack set.
+Stable pack bodies remain in their immutable sources; maintenance compacts a
+bounded suffix instead of rewriting the complete repository.
+_Avoid_: Snapshot, when referring to the stored capsule-protocol artifact

crab/Cargo.toml

@@ -72,8 +72,8 @@ tier = ["tier-s3", "tier-gcs", "tier-azure"]
 
 # Per-provider lifecycle/restore SDKs (object_store lacks these APIs).
 tier-s3 = ["dep:aws-config", "dep:aws-credential-types", "dep:aws-sdk-s3"]
-tier-gcs = ["dep:google-cloud-storage"]
-tier-azure = ["dep:azure_core", "dep:azure_mgmt_storage", "dep:azure_storage", "dep:azure_storage_blobs"]
+tier-gcs = ["dep:google-cloud-storage", "dep:google-cloud-token"]
+tier-azure = ["dep:azure_core", "dep:azure_identity", "dep:azure_mgmt_storage", "dep:azure_storage", "dep:azure_storage_blobs"]
 
 # Managed active-active coordinator control planes.
 coordinator-dynamodb = ["dep:aws-config", "dep:aws-sdk-dynamodb", "crab-coordination/coordinator-dynamodb"]

crab/docs/architecture/crab-s3-gateway.md

@@ -598,14 +598,27 @@ verified data; authorization and mutable ref resolution are not permanently
 cached along with it. Missing path visibility follows the selected permission
 contract, including distinctions between access denial and absence.
 
-The implemented repository read view is keyed by both compacted generation and
-the validated committed-journal digest. Concurrent refreshes and branch-tip
+The implemented repository read view selects a present capsule-protocol root as
+the exclusive authority and keys it by root generation plus the authenticated
+per-ref state digest. Missing v2 authority selects the canonical v1 manifest;
+malformed or incomplete v2 state fails closed without legacy fallback. V1 views
+remain keyed by compacted generation and the validated committed-journal digest.
+Concurrent refreshes and branch-tip
 snapshot resolution use singleflight cells; Git trees reuse the generation-bound
 remote-read cache, and attributes are cached per immutable commit. The gateway
 invalidates its mutable-ref view after publication. HEAD and attributed LIST
 resolve size and ETag from committed attributes without opening blob payloads.
-Committed journal packs remain readable through this path before locator/catalog
-publication completes.
+Committed journal packs and authenticated v2 capsule packs remain readable
+through their respective canonical paths before derived locator/catalog
+publication completes. For v2 repositories, gateway mutations embed the
+generated pack sidecars and exact visibility edit in one capsule, upload S3
+attributes before ref visibility, and commit through the per-ref head CAS.
+Multipart completion uses deterministic v2 intent/receipt recovery. Idle
+gateway and HTTP-server maintenance share the same root-pinned, verified
+complete-pack checkpoint implementation. Warm per-ref state tracks frontier
+length and forces a checkpoint at 56 capsules, retaining headroom below the
+hard 64-entry bound even when the write stream never becomes idle. V1 repositories retain their existing
+manifest/journal write path.
 
 GET uses logical content opening for Git, Crab and LFS content. Raw `read_blob`
 is not a substitute. Read symlinks/submodules only according to phase 0; never

crab/docs/architecture/git-capability-matrix.json

@@ -204,12 +204,11 @@
       "mirror-equal-baseline-passes-CI-policy",
       "mirror-verifies-real-origin-pointer-bytes",
       "mirror-verifies-canonical-data-without-acceleration-writes",
-      "mirror-layout-formatting-preserves-plan-identity",
-      "mirror-invalid-layout-blocks-check-plan-and-replay",
+      "mirror-invalid-root-blocks-check-plan-and-replay",
       "mirror-oversized-header-cannot-hide-corrupt-pointer",
       "mirror-restored-cache-resumes-complete-pointer-proof",
-      "mirror-equal-plan-rejects-metadata-only-change",
-      "mirror-source-ahead-plan-rejects-metadata-only-change",
+      "mirror-equal-plan-rejects-changed-snapshot",
+      "mirror-source-ahead-plan-remains-bound-after-no-op-repack",
       "mirror-clone-reconstructs-exact-source-bytes",
       "mirror-CI-detects-missing-hook",
       "mirror-plan-captures-source-ahead",

crab/docs/architecture/git-integration.md

@@ -404,10 +404,12 @@ locator and all-object visibility coverage. The accepted forms are
 repeated/combine intersections; see the support table above for semantics.
 RustFS lifecycle qualification is green; AWS/provider and released-artifact
 qualification remain before this is a released support claim.
-The planner authorizes raw OIDs from that immutable proof before reading bytes;
-the local helper produces a standard Git pack, and Git owns its promisor config,
-pack installation, and `.promisor` sidecars. A later `git cat-file`, checkout,
-diff, or merge can request missing blobs through a new helper session.
+The planner authorizes raw OIDs from that immutable proof before reading bytes.
+Git owns the repository's promisor/filter configuration and installs the
+initial protocol-v2 response. A later `git cat-file`, checkout, diff, or merge
+re-enters the line-oriented helper with raw OIDs; Crab pins the authenticated
+capsule view, generates only the authorized selection, and atomically installs
+the pack and `.promisor` sidecar into Git's object database.
 
 The `crab clone` wrapper's default lazy mode is different: it configures Crab's
 pointer checkout and does not request a Git partial clone. Use ordinary Git

crab/docs/architecture/git-protocol-v2.md

@@ -68,7 +68,9 @@ authorization boundary and is not part of the bucket-only deployment promise.
 
 The helper advertises `stateless-connect` only after it can open a single
 manifest generation with matching pack-index, locator, and all-object
-visibility coverage. The session supports:
+visibility coverage. Verified legacy v1 manifests use this same terminal wire
+with `capsule_root` absent; that transport compatibility does not create or
+select a v2 authority. The session supports:
 
 - protocol-v2 capability advertisement;
 - `ls-refs` with ref prefixes, symrefs, peeled tags, unborn HEAD, and hidden
@@ -116,9 +118,9 @@ Fresh, unfiltered fetches of exact visible ref targets use the visibility
 proof's complete per-ref closure directly. Each monotonic ref update retains a
 bounded transition from recent prior tips to the current tip, so an unfiltered
 incremental fetch can select the proven `want - have` closure without walking
-the complete object graph. A rewrite, deletion, missing transition, shallow or
-depth request, filter, or want that is not an exact ref target uses the bounded
-traversal planner. Pack generation reads up to the operation's default
+the complete object graph. A rewrite or deletion without an exact transition,
+shallow or depth request, filter, or want that is not an exact ref target uses
+the bounded traversal planner. Pack generation reads up to the operation's default
 10,000-object bound as one locator batch so adjacent pack ranges can be
 coalesced; fetched-byte and inflated-byte budgets remain the memory and I/O
 bounds. Locator batches spanning at least one exact-read wave and at least half
@@ -127,6 +129,14 @@ requested SHA-1 range. The scan abandons itself and returns to exact reads if
 stale rows would make it examine more than twice the requested object count,
 so sparse and stale-heavy repositories remain bounded.
 
+For a layered capsule's ordinary unfiltered fetch, the same authenticated
+transition chain can end negotiation as soon as it covers every visible want
+from client haves. The helper sends `ready` with the pack, but no individual
+ACK for a historical have that may no longer be visible. Hidden, unknown,
+ambiguous, or incomplete chains do not grant early completion; shallow,
+filtered, and tag-expanded requests retain their existing complete-admission
+path. The response still requires full pack planning and budget checks.
+
 For fresh `blob:none` and `object:type` requests, the catalog visibility
 bitmap is consumed as ordinals. Crab reads the additive ordinal metadata
 sidecar, filters by the published object kinds, and resolves only retained
@@ -196,10 +206,13 @@ helper leaves a bounded TTL lease for reclamation. This bounds aggregate
 provider pressure across helpers while retaining the per-process remote-Git
 object and range-read budgets.
 
-Git owns the local promisor lifecycle: the Git version in use records the
-remote's promisor/filter configuration and marks received promisor packs with
-`.promisor` sidecars. Crab's helper does not invent a second local repository
-configuration or pack-installation protocol.
+Git owns the local promisor/filter configuration and installs the initial
+protocol-v2 response. For a later raw-OID request, Git re-enters Crab through
+the line-oriented helper fetch command. Crab pins one authenticated capsule
+view, authorizes the OID against visible-ref closure, generates the selected
+pack, and uses Git's standard pack layout with an atomically installed
+`.promisor` sidecar; it does not invent a second repository configuration or
+object format.
 
 Rollback qualification accepts one of two explicit outcomes from the
 immediately prior Crab binary: it services an authorized promised raw OID with