diff --git a/.github/workflows/write-capacity.yml b/.github/workflows/write-capacity.yml new file mode 100644 index 0000000..1101b03 --- /dev/null +++ b/.github/workflows/write-capacity.yml @@ -0,0 +1,147 @@ +name: Cell write capacity qualification + +on: + pull_request: + paths: + - ".github/workflows/write-capacity.yml" + - "Cargo.toml" + - "Cargo.lock" + - ".cargo/**" + - "crates/cellule-app/**" + - "crates/cellule-host/**" + - "crates/cellule-runtime/**" + - "crates/cellule-ltx/**" + - "crates/cellule-store/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: cell-write-capacity-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + object-proof: + runs-on: ubuntu-24.04 + timeout-minutes: 100 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - name: Pin one source and release binary + run: | + set -euo pipefail + build_state="$RUNNER_TEMP/cell-write-capacity-build" + printf 'CELLULE_CAPACITY_BUILD_STATE=%s\n' "$build_state" >> "$GITHUB_ENV" + mkdir -p "$build_state/source" "$build_state/target-linux" "$build_state/evidence" + git archive HEAD | tar -x -C "$build_state/source" + git rev-parse HEAD > "$build_state/evidence/source-revision.txt" + uname -a > "$build_state/evidence/host.txt" + docker info --format '{{json .}}' > "$build_state/evidence/docker-info.json" + python3 -B -m unittest discover \ + -s crates/cellule-app/qualification -p 'test_*.py' + CELLULE_REFERENCE_STATE="$build_state" \ + docker compose -p cell-write-capacity-build \ + -f "$build_state/source/crates/cellule-app/qualification/compose.yaml" \ + run --name cell-write-capacity-build build \ + 2>&1 | tee "$build_state/evidence/build.log" + binary=$(find "$build_state/target-linux/release/deps" -maxdepth 1 \ + -type f -executable -name 'integration-*') + test -n "$binary" + test "$(printf '%s\n' "$binary" | wc -l)" -eq 1 + sha256sum "$binary" > "$build_state/evidence/binary.sha256" + - name: Run three fresh object-proof capacity repeats + run: | + set -euo pipefail + for repeat in 1 2 3; do + run_state="$RUNNER_TEMP/cell-write-capacity-run-$repeat" + mkdir -p "$run_state/evidence" + ln -s "$CELLULE_CAPACITY_BUILD_STATE/source" "$run_state/source" + ln -s "$CELLULE_CAPACITY_BUILD_STATE/target-linux" "$run_state/target-linux" + cp "$CELLULE_CAPACITY_BUILD_STATE/evidence/source-revision.txt" "$run_state/evidence/" + cp "$CELLULE_CAPACITY_BUILD_STATE/evidence/host.txt" "$run_state/evidence/" + cp "$CELLULE_CAPACITY_BUILD_STATE/evidence/docker-info.json" "$run_state/evidence/" + chmod 1777 "$run_state/evidence" + python3 "$run_state/source/crates/cellule-app/qualification/scale.py" \ + --state "$run_state" --project "cell-write-capacity-r$repeat" \ + --workload capacity + done + - name: Retain provider image identity + if: always() + run: | + docker image ls -a --digests --no-trunc \ + > "$CELLULE_CAPACITY_BUILD_STATE/evidence/images.txt" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: cell-write-capacity-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/cell-write-capacity-build/evidence/ + ${{ runner.temp }}/cell-write-capacity-run-*/evidence/ + if-no-files-found: error + retention-days: 7 + + follower-proof: + runs-on: ubuntu-24.04 + timeout-minutes: 100 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + - name: Pin follower lane source and release binary + run: | + set -euo pipefail + build_state="$RUNNER_TEMP/cell-follower-capacity-build" + printf 'CELLULE_FOLLOWER_BUILD_STATE=%s\n' "$build_state" >> "$GITHUB_ENV" + mkdir -p "$build_state/source" "$build_state/target-linux" "$build_state/evidence" + git archive HEAD | tar -x -C "$build_state/source" + git rev-parse HEAD > "$build_state/evidence/source-revision.txt" + uname -a > "$build_state/evidence/host.txt" + docker info --format '{{json .}}' > "$build_state/evidence/docker-info.json" + python3 -B -m unittest discover \ + -s crates/cellule-app/qualification -p 'test_*.py' + CELLULE_REFERENCE_STATE="$build_state" \ + docker compose -p cell-follower-capacity-build \ + -f "$build_state/source/crates/cellule-app/qualification/compose.yaml" \ + pull --quiet build rustfs bucket-init + CELLULE_REFERENCE_STATE="$build_state" \ + docker compose -p cell-follower-capacity-build \ + -f "$build_state/source/crates/cellule-app/qualification/compose.yaml" \ + run --name cell-follower-capacity-build build \ + 2>&1 | tee "$build_state/evidence/build.log" + binary=$(find "$build_state/target-linux/release/deps" -maxdepth 1 \ + -type f -executable -name 'integration-*') + test -n "$binary" + test "$(printf '%s\n' "$binary" | wc -l)" -eq 1 + sha256sum "$binary" > "$build_state/evidence/binary.sha256" + - name: Run three fresh follower-proof capacity repeats + run: | + set -euo pipefail + for repeat in 1 2 3; do + run_state="$RUNNER_TEMP/cell-follower-capacity-run-$repeat" + mkdir -p "$run_state/evidence" + ln -s "$CELLULE_FOLLOWER_BUILD_STATE/source" "$run_state/source" + ln -s "$CELLULE_FOLLOWER_BUILD_STATE/target-linux" "$run_state/target-linux" + cp "$CELLULE_FOLLOWER_BUILD_STATE/evidence/source-revision.txt" "$run_state/evidence/" + cp "$CELLULE_FOLLOWER_BUILD_STATE/evidence/host.txt" "$run_state/evidence/" + cp "$CELLULE_FOLLOWER_BUILD_STATE/evidence/docker-info.json" "$run_state/evidence/" + chmod 1777 "$run_state/evidence" + python3 "$run_state/source/crates/cellule-app/qualification/scale.py" \ + --state "$run_state" --project "cell-follower-capacity-r$repeat" \ + --workload capacity-follower + done + - name: Retain provider image identity + if: always() + run: | + docker image ls -a --digests --no-trunc \ + > "$CELLULE_FOLLOWER_BUILD_STATE/evidence/images.txt" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: cell-follower-capacity-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ runner.temp }}/cell-follower-capacity-build/evidence/ + ${{ runner.temp }}/cell-follower-capacity-run-*/evidence/ + if-no-files-found: error + retention-days: 7 diff --git a/Cargo.lock b/Cargo.lock index 1800c73..54de4bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -224,6 +224,7 @@ version = "0.1.0" dependencies = [ "async-trait", "blake3", + "bytes", "cellule-host", "cellule-ltx", "cellule-runtime", @@ -231,6 +232,7 @@ dependencies = [ "ed25519-dalek", "futures-util", "object_store", + "prost", "tempfile", "tokio", "tokio-util", diff --git a/crates/cellule-app/Cargo.toml b/crates/cellule-app/Cargo.toml index 962302c..bad0f05 100644 --- a/crates/cellule-app/Cargo.toml +++ b/crates/cellule-app/Cargo.toml @@ -16,6 +16,7 @@ cellule-runtime.workspace = true [dev-dependencies] async-trait.workspace = true +bytes.workspace = true cellule-host.workspace = true cellule-runtime = { workspace = true, features = ["test-support"] } cellule-ltx = { workspace = true, features = ["replica"] } @@ -23,6 +24,7 @@ cellule-store.workspace = true ed25519-dalek.workspace = true futures-util.workspace = true object_store.workspace = true +prost.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread"] } tokio-util.workspace = true diff --git a/crates/cellule-app/PERFORMANCE.md b/crates/cellule-app/PERFORMANCE.md index c8ee6a5..ccd736a 100644 --- a/crates/cellule-app/PERFORMANCE.md +++ b/crates/cellule-app/PERFORMANCE.md @@ -13,6 +13,8 @@ and [qualification runner](qualification/run.sh) for a fresh run. | Local typed action | [Quickstart](../../docs/quickstart.md) | Write, publish, read-back. | | Three-process RustFS | [`qualification/run.sh`](qualification/run.sh) | Local and forwarded gateway calls, receipts, drained sessions. | | Entity fleet and scaling | [`qualification/entities.py`](qualification/entities.py), [`scale.py`](qualification/scale.py) | Isolated Cell ledgers and bounded traffic. | +| Fixed 12-Cell write capacity | [`qualification/scale.py`](qualification/scale.py) with `--workload capacity` | Hot, uniform, and skewed offered-rate ramps with receipt and overload checks. | +| Fixed 12-Cell follower-proof capacity | [`qualification/scale.py`](qualification/scale.py) with `--workload capacity-follower` | Networked follower proofs, offered-rate ramps, and final root coverage. | | Reader and rollout variants | [Application integration suite](tests/integration.rs) | Selection, replacement, and recovered receipts. | ```mermaid @@ -38,6 +40,7 @@ runs do not establish production capacity. | What did the public action and deployment probes show? | [Action](performance/2026-09-25-public-host-action.md), [RustFS](performance/2026-09-25-public-host-rustfs.md), [three-node Compose](performance/2026-09-27-three-node-compose.md), [replica Compose](performance/2026-09-27-replica-compose.md) | | How were reader recruitment and publication evaluated? | [Recruitment](performance/2026-09-27-reader-recruitment.md), [publication](performance/2026-09-27-reader-publication.md), [expiry](performance/2026-09-27-reader-expiry.md), [loss during load](performance/2026-09-27-reader-loss-during-load.md) | | What were the scale and mixed-load limits? | [Scaling](performance/2026-09-27-reader-scaling.md), [mixed readers](performance/2026-09-27-mixed-readers.md), [host readers Compose](performance/2026-09-27-host-readers-compose.md), [qualification notes](performance/2026-09-27-qualification-notes.md) | +| Is the writable Cell limit established? | [Write capacity measurement status](performance/2026-09-29-write-capacity.md) | ```mermaid flowchart LR diff --git a/crates/cellule-app/performance/2026-09-29-write-capacity.md b/crates/cellule-app/performance/2026-09-29-write-capacity.md new file mode 100644 index 0000000..51be463 --- /dev/null +++ b/crates/cellule-app/performance/2026-09-29-write-capacity.md @@ -0,0 +1,380 @@ +# Writable entity capacity: measurement status, 2026-09-29 + +The existing ignored `entity_process_scaling` integration workload schedules +uniform, hot, and skewed traffic at 1, 4, and 16 actions per node per second +across 3, 5, 10, and 20 constrained nodes. Its independent parser checks all +arrivals, stable mutation identities, acknowledged receipts, readback values, +owner records, and final published roots. An integrity pass means that the +reported results are internally consistent; it does not certify a supported +rate when arrivals were missed. + +This revision adds separate raw command-response proof and object-publication +completion records per node. `node-N-responses.tsv` identifies Recorded, +Fleet, or Object as the single proof that released each successful runtime +command response. `node-N-publications.tsv` records queue wait, root +preparation, authority/confirmation time, and total background publication +time per commit sequence. The parser reports their distributions and published +roots/s separately from completed client actions/s. It also reports an +arrival-latency distribution over every scheduled action, including rejected +and late arrivals. The earlier resource, logical object-operation, receipt, +and readback files remain required. + +A subsequent instrumentation revision adds `node-N-executions.tsv` with actor +queue wait and SQL worker round trip per attempted command. This observation +ends before durability submission and proof. The verifier reports both +distributions per window, which helps distinguish owner admission and worker +occupancy from storage publication without treating either as a durable +response. The first CI result below predates this file. + +The new `--workload capacity` selector fixes the fleet at three owners and +12 writable Cells. The initial CI run scheduled 10-second points at 2, 4, 16, +64, 256, and 1024 actions per node per second. The second run added 24, 32, +48, 96, 128, and 192 to narrow the overload interval. Each shape stops at its first overloaded +point. A point is fully served only when every scheduled action succeeds and +admitted work drains within 12 seconds of its 10-second arrival window. The +verifier requires at least one fully served point and one overloaded +point for uniform, hot, and skewed traffic; it rejects missing arrivals, +misstated success, incomplete readback, or an incomplete rate ramp. It reports +the last fully served logical write rate separately from the first overloaded +rate. The first and second isolated object-proof results are summarized below. + +The dedicated `Cell write capacity qualification` GitHub Actions workflow +builds release binaries for separate object-proof and follower-proof jobs. +Each job runs three repeats; each repeat has its own Compose project, RustFS +volume, object prefix, and evidence directory. The workflow uploads raw +samples, logs, binary digests, and provider details even if a repeat fails. +Its output requires review before a capacity claim. + +## First isolated object-proof result + +[CI run 36649534205](https://github.com/crabbuild/cellule/actions/runs/36649534205) +passed three fresh-provider repeats on the same binary, with 12 Cells and +readback verification in every run. The source snapshot was +`4837fc823c198f51a85b01528a6c7aaf6b1f4470`; the release integration +binary SHA-256 was +`2b3aa0b36df5d2c278fe2f3e524a6258c5f886ddedab927057bd4c8db76834a3`. +The RustFS image was pinned at +`sha256:bffcab0c9d647aab0055d1c69d340b202d0909966b385932d4ead1aeb7602858`. +All response proofs were Object; each verified window reported zero root +sequence lag at its end. + +| Shape | Highest fully served offered rate | Fully served logical writes/s | First overloaded offered rate | Fully served action p95 / p99 across repeats | +| --- | ---: | ---: | ---: | ---: | +| Uniform writes | 16 actions/node/s | 47.99–48.00 | 64 actions/node/s | 17.68–32.05 / 120.89–161.14 ms | +| One hot writable Cell | 16 actions/node/s | 47.99–48.00 | 64 actions/node/s | 40.84–52.06 / 56.45–76.39 ms | +| Skewed, 20% writes | 64 actions/node/s | 38.39–38.40 | 256 actions/node/s | 18.26–20.21 / 36.98–43.24 ms | + +These are tested lower bounds, not precise saturation points. The overloaded +uniform windows had client concurrency rejection and owner capacity refusals; +hot windows had 976–982 owner capacity refusals plus 47–55 writes whose +receipt-bound read failed; skewed windows had 1,911–2,227 scheduler-late +arrivals and 469–667 read failures. The upper rate cannot be reported as +sustainable even though some writes completed. No node showed cgroup CPU +throttling. Capture p95 stayed below 2 ms in overloaded windows, whereas +root preparation and response waits were much longer. The current aggregate +provider counters cannot isolate GET/HEAD/PUT time inside preparation or +separate owner admission queueing from provider wait. The second run below +adds raw per-operation provider timing. + +The three `verification.json` files have SHA-256 digests +`35b859c9f9f66aa46da1c797fae5d691a52af907760cf3c4bacc7d79ec809fe2`, +`8a3b951a95a1b9e2b2b00ab8954df53c2783acf352cf529f684fc00be2cd6111`, +and `9d13723e28d4989a87dc60d88b7c84780c125d716ef1d3b1504eb1895fd9efc7` +for repeats 1–3 respectively. Each report contains hashes of its raw TSVs. +The downloaded artifact is under +`$HOME/Workspace/crabbuild-target/cellule-capacity-5ca5/ci-run-36649534205`. + +## Finer object-proof result + +[CI run 36651191247](https://github.com/crabbuild/cellule/actions/runs/36651191247) +passed three fresh-provider repeats with all acknowledged writes verified by +readback. The source snapshot was +`f46c98c66a78c84bac2244eb739d7548d4ceb056`; the release binary SHA-256 +was `b75352d08f379318f0a872fce7286d13590715eb49aecf7d53cc2cc7906bca7a`. +All response proofs were Object, every window ended with zero root sequence +lag, and no node reported CPU throttling. + +| Shape | Highest fully served offered rate across repeats | Fully served logical writes/s | First overloaded offered rate | +| --- | --- | --- | --- | +| Uniform writes | 24, 32, 32 actions/node/s | 71.96, 95.89, 95.88 | 32, 48, 48 actions/node/s | +| One hot writable Cell | 24, 24, 16 actions/node/s | 71.99, 71.92, 47.99 | 32, 32, 24 actions/node/s | +| Skewed, 20% writes | 24, 24, 16 actions/node/s | 14.40, 14.40, 9.60 | 32, 32, 24 actions/node/s | + +The threshold varies between repeats. Uniform overload mixed scheduler-late +arrivals with owner refusals in two repeats; hot overload mainly returned +owner `not_started` capacity refusals. Skewed overload was only 3–5 +scheduler-late arrivals despite low CPU use, so it is a harness scheduling +limit, not evidence of a Cell write limit. At hot overload, owner 0 response +p95 was 77–169 ms while publication p95 was 36–42 ms and capture p95 stayed +under 1 ms. Its provider PUT p95 was 6.7–7.5 ms, GET p95 1.9–2.1 ms, and +HEAD p95 1.4–1.6 ms. Provider operations can overlap, so these percentiles +cannot be added to infer one command's critical path. The gap between owner +response and publication requires the actor queue and SQL worker timings added +after this run before choosing a write-path optimization. + +The three `verification.json` SHA-256 digests are +`1624b007bd18b19b0e5b498a4cdc307fcfdc30a2320c2ebbd0271f097838c31f`, +`11d03fdc4835bd00a9fc0c90211cb9fd3a21d543ef4f91dc9abb4a64dd15d51c`, +and `d0e41bf722a143149742c47477885d7ea4442a7717d09f7cb6083c916df03e43`. +The downloaded artifact is under +`$HOME/Workspace/crabbuild-target/cellule-capacity-5ca5/ci-run-36651191247`. + +## Actor and worker timing attempt + +[CI run 36653277555, attempt 1](https://github.com/crabbuild/cellule/actions/runs/36653277555) +passed readback and evidence integrity in all three repeats. The source +snapshot was `7e1c89c20d62b403ebcace3d208381bcfdfd6485`; the binary SHA-256 +was `d8778c143fe99b9ff5ff2a4c619bba8f7e8a3cdd7548d1d3a92d6fd64fe22880`. +All response proofs were Object and root lag was zero at window ends. The +fully served uniform rate varied from 2 to 16 actions/node/s, while the first +overloaded skewed rate ranged from 4 to 16. Some windows stopped after a +single scheduler-late arrival at 4 actions/node/s with 0.8–2.1% node CPU use +and no CPU throttling. Worker, publication, and provider tail timings also +spiked at these low rates. This attempt does not yield a stable saturation +point or one repeatable dominant phase. A same-revision rerun on a fresh CI +runner is required before selecting a write-path change. +Attempt 2 built the same revision but could not start the first repeat: +the pinned `bucket-init` image pull returned `toomanyrequests: Data limit +exceeded` from its public registry. It produced no capacity measurements. +The docs-only rerun on source `f9b918a` ([CI run +36657593245](https://github.com/crabbuild/cellule/actions/runs/36657593245), +attempts 1 and 2) failed at the same image pull before traffic. The +qualification fixture now pins the same AWS CLI 2.27.41 version from Docker +Hub at manifest digest +`sha256:bc6b7bba44ce38f9604ede49c584824af919047ea03fbcc7c7610671fdef95d8`; +the object-store image, resource limits, rate schedule, and verifier are +unchanged. +Its bucket-init completed against a fresh local RustFS Compose volume; the +three-repeat CI result is reported below. + +The attempt-1 `verification.json` SHA-256 digests are +`b1282ef781b4ca9d962fb43dbf2949662e589ce7737f730fd276473689483673`, +`288e0b50d79780bf4540b9ca2d988880a8d494ca47285a151d5f91964fbd257f`, +and `d9abe2dc3ca698dda71e586da118bc8af0e7fdf2b97b982d963ff4946d108e18`. +The downloaded artifact is under +`$HOME/Workspace/crabbuild-target/cellule-capacity-5ca5/ci-run-36653277555`. + +## Final hot Cell attribution + +[CI run 36655966439](https://github.com/crabbuild/cellule/actions/runs/36655966439) +passed three fresh-provider repeats on source snapshot +`3d01ab9494f76cdf7cb249098ce1814e89f2b203` and binary SHA-256 +`fe847a82c53e456b1f71c32d9309eeb6a6e8e864372a183230db9bedf86eb842`. +All acknowledged writes passed readback, all response proofs were Object, +every window ended at zero root lag, and no node reported CPU throttling. + +| Shape | Fully served offered rate across repeats | Logical writes/s | First overloaded rate | +| --- | --- | --- | --- | +| One hot writable Cell | 24, 24, 24 actions/node/s | 71.96–71.99 | 32, 32, 32 actions/node/s | +| Uniform writes | 24, 32, 32 actions/node/s | 71.96–95.92 | 32, 48, 48 actions/node/s | +| Skewed, 20% writes | 96, 64, 96 actions/node/s | 38.40–57.59 | 128, 96, 128 actions/node/s | + +At hot overload, owner 0 published 62.28–65.10 roots/s. Mean publication +time was 15.10–16.00 ms/root, with 11.64–11.69 provider requests per +acknowledged write. Actor queue p95 climbed from 49–83 ms at the fully served +point to 142–160 ms at overload, while SQL worker round-trip p95 was only +2.03–2.28 ms at overload. Publication p95 was 37–43 ms, of which root +preparation p95 was 33–37 ms. Supported hot action p95/p99 varied from +112–198/175–222 ms; overloaded action p95/p99 was 288–322/324–435 ms. +Owner `not_started` capacity refusals began at the 32 actions/node/s point. +Together these observations identify serialized object publication as the +hot Cell throughput limiter on this object-proof profile. They do not yet +distinguish predecessor verification, directory work, immutable uploads, or +compaction within preparation; that split is required before code changes. +Uniform and skewed thresholds varied between repeats, so this result is +specific to the hot shape. + +The three `verification.json` SHA-256 digests are +`40e4908a169013a80fe873f4aaf0d6f355872545a72212addb0d712281724e31`, +`58740dbf1d5a57ed16b2138c011e2f694a4900b728967dee019b32a5eb187717`, +and `7d379162a637ba97dd197a8bc85c9d3abebe584454f9d7f65a683c065fb289be`. +The downloaded artifact is under +`$HOME/Workspace/crabbuild-target/cellule-capacity-5ca5/ci-run-36655966439`. + +## Qualification image rerun + +[CI run 36659182959](https://github.com/crabbuild/cellule/actions/runs/36659182959) +passed all three object-proof repeats after changing only the bucket-init +image registry. The PR test-merge source was +`4b52795bf23cd32b72c820269562b6e3e7d8d0fc`; the release integration +binary SHA-256 was +`8b23d9b7b88c082869a9bcab20e19ebbd345c8165f5c1ea5bb38d545d9d4d850`. +All three reports passed integrity and acknowledged-write readback checks, +every response winner was Object, root lag ended at zero, and the hot owner +reported zero CPU throttling. + +| Shape | Fully served, repeats 1–3 | First overloaded, repeats 1–3 | +| --- | --- | --- | +| One hot writable Cell | 16, 16, 16 actions/node/s | 24, 24, 24 actions/node/s | +| Uniform | 24, 32, 32 actions/node/s | 32, 48, 48 actions/node/s | +| Skewed, 20% writes | 48, 48, 48 actions/node/s | 64, 64, 64 actions/node/s | + +At hot overload, owner 0 published 57.84–59.05 roots/s. Its actor queue +p95 was 136–142 ms, SQL worker round-trip p95 2.53–2.75 ms, publication +p95 37–43 ms, and root preparation p95 32–38 ms. Provider requests per +acknowledged write were 11.66–11.73. This independently supports serial +object publication as the hot Cell limiter, while the lower hot rate interval +than run 36655966439 shows that exact throughput is sensitive to the shared +CI runner. A controlled A/A baseline is required before claiming a +code-change gain. + +The `verification.json` SHA-256 digests for repeats 1–3 are +`5ff25de33a3c9edacd677b3e724b0003d28aceea1f0f2cb3e8f9b729e85d9086`, +`31035c59762a5d34eea412a52b412924f9fded01cca9085c05930a5ba347194c`, +and `bfbef7176b533f326d1e536122a814afbf6e2b7d8bbcfe3184574fcbac0b4940`. +The downloaded artifact is under +`$HOME/Workspace/crabbuild-target/cellule-capacity-5ca5/ci-run-36659182959`. + +## First networked follower-proof comparison + +[CI run 36662251677](https://github.com/crabbuild/cellule/actions/runs/36662251677) +passed both capacity jobs, each with three fresh-provider repeats and the same +release binary (`69ad461750472d181e1a326a2d4f5e6ef8ade22ca14920e9b4c4e40ad83bf45f`). +The test-merge source was `f2a5db377cbc11da01278914bc6fa1996b4b4565`. +The follower lane used a signed TCP endpoint, two private-disk followers per +owner, and the object-only lane's 12 Cells, arrivals, resource limits, and +readback rules. Every acknowledged write passed readback, and final published +roots covered all receipts. Follower response winners were Fleet/Object +2,245/24, 3,740/7, and 4,657/24 across repeats 1–3. The object lane used +only Object proofs. + +| Shape | Object fully served offered rate, repeats 1–3 | Follower fully served offered rate, repeats 1–3 | Follower first overloaded rate | +| --- | --- | --- | --- | +| Uniform writes | 16, 32, 32 actions/node/s | 4, 24, 24 actions/node/s | 16, 32, 32 | +| One hot writable Cell | 24, 16, 24 | 16, 4, 16 | 24, 16, 24 | +| Skewed, 20% writes | 48, 64, 64 | 24, 32, 48 | 32, 48, 64 | + +At the common hot rate of 16 actions/node/s, all object repeats were fully +served with action p95 of 48.06–56.90 ms. Follower repeats 1 and 3 were fully +served with p95 of 17.12–26.37 ms; repeat 2 missed four scheduled arrivals +despite p95 of 21.58 ms among successful actions. Hot follower overload at +24 actions/node/s returned 168–238 owner capacity refusals in repeats 1 and +3. The follower path reduces response latency at this matched point, but its +fully served throughput bound is lower and variable on these shared runners. +The two jobs used different runners, so repeat numbers are not paired host +measurements. + +The response and publication capacities differ. A fully served hot follower +window in repeat 1 completed about 48 writes/s while publishing 46.56 roots/s +and ended 13 commits ahead of its root. Its final root did drain and cover +every acknowledged receipt. Uniform fully served windows in repeats 2 and 3 +ended one commit ahead. These are response-supported windows; their published +roots/s and remaining lag must be read separately. The first overloaded +uniform and skewed follower points were mostly isolated scheduler-late +arrivals, so they do not establish storage saturation. + +The three follower `verification.json` SHA-256 digests are +`c927b09f8f0200b05548d02827aba28434014b8fa1dc7cea0e983f0893001e6e`, +`2e2e9a657fc200103224f2830804f9f9d629b390b91500498800cb74f38ee8b7`, +and `12f93bc42bd015f081f528504ff82f580edfe8db3c78eef7b5d1a52e53d6f75c`. +The corresponding object digests are +`7ddbbf67f22b8554738c73b73d2be76338f52deb46bb9223f1cda83dca22edca`, +`2cf37c2a34158f9e79a58850fc64873924642149fdd83b7e2d9d326adc90a24f`, +and `64c9182aa10c691a8ae32da608b9533f8a0384b02f08f6c9e8e0d3e438f83ac9`. +Raw logs, samples, provider digests, and these reports are under +`$HOME/Workspace/crabbuild-target/cellule-capacity-5ca5/ci-run-36662251677`. +The enclosing PR check failed in its separate reader smoke because that +fixture still used a shared signing key after advertisements switched to +per-node keys. The next revision corrects that mismatch and adds network +append duration evidence; this capacity result remains valid for its pinned +binary and must be followed by a clean full rerun. + +## Clean follower-proof rerun and variability + +[CI run 36663653146](https://github.com/crabbuild/cellule/actions/runs/36663653146) +passed the workspace smoke and both three-repeat capacity jobs. The +test-merge source was `74933c331e477c36f01b0fb508fa555bf897d19f`, and both +capacity lanes used release binary SHA-256 +`e4e48c7c50c8dc5a1a2c1f542fb3df6d38517e93fbc8926476c2024dcaf9c5bd`. +The source and binary were +pinned within the run; each repeat used a fresh provider volume and the same +12-Cell schedule. Every acknowledged write passed readback, every final root +covered its receipts, and no node reported CPU throttling. + +| Shape | Object fully served offered rate, repeats 1–3 | Follower fully served offered rate, repeats 1–3 | +| --- | --- | --- | +| Uniform writes | 2, 4, 4 actions/node/s | 16, 4, 4 actions/node/s | +| One hot writable Cell | 4, 4, 16 | 4, 4, 2 | +| Skewed, 20% writes | 4, 4, 4 | 2, 4, 2 | + +Most first-overload points in both lanes missed only 1–20 scheduled arrivals, +including at 4 actions/node/s. They cannot identify a stable storage throughput +limit. At the fully served hot point, object action p95 ranged from 50.42 to +147.86 ms; follower action p95 ranged from 58.33 to 155.84 ms. The new +end-to-end member append measurement had owner-0 p95 of 17.38–141.02 ms at +the follower hot fully served points. It includes member resolution, TCP, +authority lookup, and follower fsync; it does not yet attribute those phases. +The follower owner used 13.14–14.35 object-provider requests per acknowledged +hot write versus 11.12–11.69 in the object lane. This is consistent with +extra authority reads in the signed append path, but the current counters do +not prove which read or provider wait caused the tail. The first run's hot +latency gain is therefore an observed result on that pinned binary, not a +repeatable improvement across runners. A controlled A/A baseline and phase +split are needed before optimizing this transport path. + +Follower `verification.json` SHA-256 digests are +`697bb3e5f88d90a9cf6fc802ebf025a643a3cbf479b3254838e5410ada557fb7`, +`f97b12e31794e9767c48e713010f1087c684a025ee44625272b1afb0bf0362c7`, +and `28c5fcfb130f23a2f4b2842804fd64dc99a62f46f9a5225677dd8bb762e462c6`. +Object digests are +`264f2b65f81fff66ba8d87b246b551fb2840bb8859bf74aa9de415c4a3afee7f`, +`cd37d078b95f4b870db37bb22cecd455b196c29b044b5b9bfc24c078f72add6c`, +and `93c6b119f3e76a0cf1a2319703c9f0956cb2e28970d4d28b84af2bf918b521b4`. +Raw evidence and provider image digests are under +`$HOME/Workspace/crabbuild-target/cellule-capacity-5ca5/ci-run-36663653146`. + +After preparing a fresh source/binary snapshot with the Compose qualification +guide, run each repeat with a new state directory and Compose project: + +```sh +python3 "$CELLULE_REFERENCE_STATE/source/crates/cellule-app/qualification/scale.py" \ + --state "$CELLULE_REFERENCE_STATE" --project capacity-unique-run \ + --workload capacity +``` + +The object-proof-only Compose profile records LTX phases and capture timing, +publication objects and bytes, response proof, background publication, +per-Cell root sequence lag, and per-node resource and provider operations. +The resource stream also samples unpublished follower-log bytes from runtime +stats; it is expected to be zero in the object-only lane. +Root lag is derived from the latest acknowledged receipt and completed root +publication at each window's wall-clock end; clock adjustments limit its +precision. A scheduler-late or client-full +overload point identifies the harness admission limit until runtime and +provider phase evidence demonstrates a narrower Cell limiter. + +The original object-only profile could not supply a follower-enabled +comparison. The later follower-proof lane above adds a networked node-log +transport and authority enrollment; a separate process test covers owner-loss +recovery. The added publication observation aggregates root preparation and +provider I/O; it cannot by itself distinguish predecessor +GET/HEAD, immutable PUT, and provider wait inside that phase. Those limits +prevent selecting a safe publication change from this evidence alone. + +The 2026-09-29 workstation did not provide an isolated provider environment. +The Colima VM had multiple unrelated active RustFS workloads, several above +one CPU, while the Linux qualification binary was building. Running the +3/5/10/20-node profile three times there would mix provider and CPU contention +from those workloads into the Cellule curve. A raw Docker container snapshot +is retained under +`$HOME/Workspace/crabbuild-target/cellule-capacity-5ca5/state-20260929/evidence/shared-host-docker-stats.tsv`. +The VM also could not bind-mount the mounted Workspace volume; a disposable +source snapshot was therefore moved under `$HOME/.codex/qualification/` for +the Linux build. No capacity claim or bottleneck classification is made from +that build. + +The earlier disposable snapshot built the Linux `cellule-app` integration test in +release mode with `cargo test --release --locked -p cellule-app --test integration --no-run`. +Its binary SHA-256 is +`6cf7c3dc86ff673c86bcd82b2eedcafbbe2db358f4c16e280d41600476ea9983`. +The snapshot is under `$HOME/.codex/qualification/cellule-capacity-5ca5-state`; +the original source archive and host evidence are under the Workspace path +above. That binary predates the fixed-Cell capacity selector and cannot run +it; create a new source snapshot and release binary for the capacity runs. +This is a compile result, not an execution result. + +A publication optimization requires a dedicated provider environment and a +repeatable A/A rate interval before modifying publication. It then splits root +preparation into predecessor reads, directory work, uploads, provider wait, +and CAS. A follower-enabled lane runs separately, with recovery proof and +eventual root drain alongside response throughput. diff --git a/crates/cellule-app/qualification/compose.yaml b/crates/cellule-app/qualification/compose.yaml index d596d96..9feb142 100644 --- a/crates/cellule-app/qualification/compose.yaml +++ b/crates/cellule-app/qualification/compose.yaml @@ -72,7 +72,7 @@ services: restart: "no" bucket-init: - image: public.ecr.aws/aws-cli/aws-cli:2.27.41@sha256:1c2d7a51b1ff4f460bc3f1e1ee46a6cef47c7429ad9089ef99012a95e472a1a5 + image: amazon/aws-cli:2.27.41@sha256:bc6b7bba44ce38f9604ede49c584824af919047ea03fbcc7c7610671fdef95d8 environment: *storage command: [--endpoint-url, http://rustfs:9000, s3api, create-bucket, --bucket, cellule-reference-app] depends_on: diff --git a/crates/cellule-app/qualification/entities.py b/crates/cellule-app/qualification/entities.py index 38d63dd..ccec905 100644 --- a/crates/cellule-app/qualification/entities.py +++ b/crates/cellule-app/qualification/entities.py @@ -3,6 +3,7 @@ from __future__ import annotations import bisect +from collections import Counter import csv import hashlib from pathlib import Path @@ -10,8 +11,12 @@ STAGES = (3, 5, 10, 20) SHAPES = ("uniform", "hot", "skewed") POINTS = ((1, 4), (4, 16), (16, 64)) +CAPACITY_POINTS = ((2, 8), (4, 16), (16, 64), (24, 96), (32, 128), + (48, 192), (64, 256), (96, 256), (128, 256), + (192, 256), (256, 256), (1024, 256)) CELLS_PER_NODE = 4 SECONDS = 10 +CAPACITY_DRAIN_GRACE_US = 2_000_000 def rows(path: Path) -> list[dict]: @@ -38,9 +43,154 @@ def distribution(values: list[int]) -> dict: }) +def verify_object_operations(control: Path, node: int, observations: list[dict]) -> list[dict]: + samples = rows(control / f"node-{node}-object-operations.tsv") + assert Counter((row["operation"], row["outcome"]) for row in samples) == { + (row["operation"], row["outcome"]): int(row["count"]) for row in observations + if int(row["count"]) > 0}, "object operation samples disagree with counters" + for row in samples: + assert int(row["at_ms"]) > 0 + assert all(int(row[key]) >= 0 for key in ("duration_us", "bytes_read", "bytes_written")) + return samples + + +def verify_timing_evidence(control: Path, node: int, windows: list[dict]) -> dict: + responses = rows(control / f"node-{node}-responses.tsv") + executions = rows(control / f"node-{node}-executions.tsv") + publications = rows(control / f"node-{node}-publications.tsv") + phases = rows(control / f"node-{node}-phases.tsv") + captures = rows(control / f"node-{node}-captures.tsv") + costs = rows(control / f"node-{node}-publication-costs.tsv") + appends = rows(control / f"node-{node}-follower-appends.tsv") + network = rows(control / f"node-{node}-follower-network.tsv") + log_events = rows(control / f"node-{node}-node-log-events.tsv") + assert responses, f"node {node}: missing command response evidence" + assert executions, f"node {node}: missing command execution evidence" + assert publications, f"node {node}: missing publication evidence" + assert phases and captures and costs, f"node {node}: missing LTX or publication phase evidence" + sources = {"Recorded", "Fleet", "Object"} + for row in responses: + assert row["source"] in sources + assert int(row["at_ms"]) > 0 + assert int(row["response_us"]) >= int(row["confirmation_us"]) >= 0 + for row in executions: + assert int(row["at_ms"]) > 0 + assert int(row["queue_wait_us"]) >= 0 and int(row["worker_round_trip_us"]) >= 0 + assert row["succeeded"] in {"true", "false"} + seen = set() + for row in publications: + key = (row["cell"], int(row["sequence"])) + assert key not in seen, f"node {node}: duplicate publication" + seen.add(key) + assert int(row["at_ms"]) > 0 and int(row["sequence"]) > 0 + assert row["succeeded"] in {"true", "false"} + total = int(row["total_us"]) + assert total >= 0 + for phase in ("queue_wait_us", "preparation_us", "authority_us"): + assert 0 <= int(row[phase]) <= total + for row in phases: + assert int(row["at_ms"]) > 0 and int(row["elapsed_us"]) >= 0 + assert row["succeeded"] in {"true", "false"} + for row in captures: + assert int(row["at_ms"]) > 0 and row["succeeded"] in {"true", "false"} + total = int(row["total_us"]) + assert total >= 0 + for key in ("preparation_us", "schema_check_us", "wal_read_us", "page_collection_us", + "verification_us", "encode_us", "local_write_us", "fsync_us", "checkpoint_us"): + assert 0 <= int(row[key]) <= total + assert int(row["wal_bytes"]) >= 0 and int(row["ltx_bytes"]) >= 0 + for row in costs: + assert int(row["at_ms"]) > 0 and int(row["objects"]) >= 0 and int(row["bytes"]) >= 0 + for row in appends: + assert int(row["at_ms"]) > 0 and int(row["bytes"]) >= 0 + assert row["acknowledged"] in {"true", "false"} + for row in network: + assert int(row["at_ms"]) > 0 + assert int(row["bytes"]) >= 0 and int(row["duration_us"]) >= 0 + assert row["acknowledged"] in {"true", "false"} + last_covered = 0 + for row in log_events: + assert int(row["at_ms"]) > 0 and int(row["epoch"]) > 0 + assert row["phase"] in {"enrolled", "active", "coverage", "closed"} + covered = int(row["covered_through"]) + if row["phase"] in {"enrolled", "active"}: + assert covered == 0, "node-log lifecycle marker has coverage" + else: + assert covered >= last_covered, "node-log coverage regressed" + last_covered = covered + for window in windows: + if node >= window["nodes"]: + continue + start, end = window["started_ms"], window["ended_ms"] + selected_responses = [row for row in responses if start <= int(row["at_ms"]) <= end] + selected_executions = [row for row in executions if start <= int(row["at_ms"]) <= end] + selected_publications = [row for row in publications if start <= int(row["at_ms"]) <= end] + selected_phases = [row for row in phases if start <= int(row["at_ms"]) <= end] + selected_captures = [row for row in captures if start <= int(row["at_ms"]) <= end] + selected_costs = [row for row in costs if start <= int(row["at_ms"]) <= end] + selected_appends = [row for row in appends if start <= int(row["at_ms"]) <= end] + selected_network = [row for row in network if start <= int(row["at_ms"]) <= end] + covered_before_end = [int(row["covered_through"]) for row in log_events + if row["phase"] in {"coverage", "closed"} and int(row["at_ms"]) <= end] + response_sources = {source: sum(row["source"] == source for row in selected_responses) + for source in sorted(sources)} + window.setdefault("node_durability", {})[node] = dict( + response_sources=response_sources, + response_latency=distribution([int(row["response_us"]) for row in selected_responses]), + confirmation_latency=distribution([int(row["confirmation_us"]) for row in selected_responses]), + actor_queue=distribution([int(row["queue_wait_us"]) for row in selected_executions]), + worker_round_trip=distribution([int(row["worker_round_trip_us"]) for row in selected_executions]), + worker_failures=sum(row["succeeded"] == "false" for row in selected_executions), + publication_total=distribution([int(row["total_us"]) for row in selected_publications]), + publication_queue=distribution([int(row["queue_wait_us"]) for row in selected_publications]), + publication_preparation=distribution([int(row["preparation_us"]) for row in selected_publications]), + publication_authority=distribution([int(row["authority_us"]) for row in selected_publications]), + published_roots_per_second=sum(row["succeeded"] == "true" for row in selected_publications) + * 1_000_000 / window["elapsed_us"], + latest_published_sequence_by_cell={cell: max(int(row["sequence"]) for row in publications + if row["cell"] == cell and row["succeeded"] == "true" + and int(row["at_ms"]) <= end) + for cell in {row["cell"] for row in publications + if row["succeeded"] == "true" and int(row["at_ms"]) <= end}}, + failed_publications=sum(row["succeeded"] == "false" for row in selected_publications)) + window["node_durability"][node].update( + ltx_phases={phase: distribution([int(row["elapsed_us"]) for row in selected_phases + if row["phase"] == phase]) + for phase in sorted({row["phase"] for row in selected_phases})}, + capture_total=distribution([int(row["total_us"]) for row in selected_captures]), + uploaded_objects=sum(int(row["objects"]) for row in selected_costs), + uploaded_bytes=sum(int(row["bytes"]) for row in selected_costs), + follower_appends=len(selected_appends), + follower_append_failures=sum(row["acknowledged"] == "false" for row in selected_appends), + follower_append_bytes=sum(int(row["bytes"]) for row in selected_appends), + follower_network_latency=distribution([int(row["duration_us"]) for row in selected_network]), + follower_network_bytes=sum(int(row["bytes"]) for row in selected_network), + node_log_covered_through=max(covered_before_end, default=0)) + return dict(response_sources={source: sum(row["source"] == source for row in responses) + for source in sorted(sources)}, + response_latency=distribution([int(row["response_us"]) for row in responses]), + actor_queue=distribution([int(row["queue_wait_us"]) for row in executions]), + worker_round_trip=distribution([int(row["worker_round_trip_us"]) for row in executions]), + publication_total=distribution([int(row["total_us"]) for row in publications]), + capture_total=distribution([int(row["total_us"]) for row in captures]), + uploaded_objects=sum(int(row["objects"]) for row in costs), + uploaded_bytes=sum(int(row["bytes"]) for row in costs), + follower_appends=len(appends), + acknowledged_follower_appends=sum(row["acknowledged"] == "true" for row in appends), + follower_append_bytes=sum(int(row["bytes"]) for row in appends), + acknowledged_network_appends=sum(row["acknowledged"] == "true" for row in network), + follower_network_latency=distribution([int(row["duration_us"]) for row in network]), + node_log_phases=dict(Counter(row["phase"] for row in log_events)), + node_log_epochs=sorted({int(row["epoch"]) for row in log_events}), + node_log_covered_through=last_covered, + completed_publications=sum(row["succeeded"] == "true" for row in publications), + failed_publications=sum(row["succeeded"] == "false" for row in publications)) + + def verify_window(control: Path, nodes: int, shape: str, rate_per_node: int, - concurrency: int, window_id: int, positions: dict[int, list[int]]) -> dict: - label = f"entities-{nodes}-{shape}-{rate_per_node}" + concurrency: int, window_id: int, positions: dict[int, list[int]], + prefix: str = "entities") -> dict: + label = f"{prefix}-{nodes}-{shape}-{rate_per_node}" metadata, = rows(control / f"{label}-window.tsv") assert metadata["shape"] == shape for key, expected in dict(window_id=window_id, nodes=nodes, rate_per_node=rate_per_node, @@ -53,7 +203,7 @@ def verify_window(control: Path, nodes: int, shape: str, rate_per_node: int, planned = rate * SECONDS samples = sorted(rows(control / f"{label}.tsv"), key=lambda row: int(row["arrival"])) assert [int(row["arrival"]) for row in samples] == list(range(planned)), "missing or duplicate arrival" - successes, arrival_latencies, writes = [], [], [0] * nodes + successes, arrival_latencies, scheduled_latencies, writes, actions = [], [], [], [0] * nodes, [0] * nodes outcomes, intervals = {}, [] new_positions = {entity: [] for entity in range(nodes * CELLS_PER_NODE)} for sample in samples: @@ -69,6 +219,7 @@ def verify_window(control: Path, nodes: int, shape: str, rate_per_node: int, assert (entity, sample["kind"]) == destination(shape, arrival, nodes * CELLS_PER_NODE) assert outcome in {"ok", "resolved", "write_only", "not_started", "absent", "client_full", "scheduler_late", "read_failed"} outcomes[outcome] = outcomes.get(outcome, 0) + 1 + scheduled_latencies.append(started + elapsed - scheduled) if outcome in ("client_full", "scheduler_late"): assert elapsed == sequence == read_sequence == count == 0 if outcome == "scheduler_late": @@ -85,6 +236,7 @@ def verify_window(control: Path, nodes: int, shape: str, rate_per_node: int, assert read_sequence >= max(sequence, 1) assert sequence > 0 if sample["kind"] == "write" else sequence == 0 successes.append(elapsed) + actions[entity // CELLS_PER_NODE] += 1 arrival_latencies.append(started + elapsed - scheduled) else: assert read_sequence == count == 0 @@ -113,18 +265,78 @@ def verify_window(control: Path, nodes: int, shape: str, rate_per_node: int, return dict(nodes=nodes, shape=shape, rate_per_node=rate_per_node, concurrency=concurrency, planned=planned, outcomes=outcomes, fully_served_arrivals=len(successes) == planned, acknowledged_writes_by_node=writes, completed_actions=len(successes), + completed_actions_by_node=actions, + latest_write_sequence_by_entity={entity: max(values, default=0) + for entity, values in positions.items()}, + completed_writes_per_second=sum(writes) * 1_000_000 / elapsed_us, completed_per_second=len(successes) * 1_000_000 / elapsed_us, service_latency=distribution(successes), arrival_latency=distribution(arrival_latencies), + scheduled_arrival_latency=distribution(scheduled_latencies), started_ms=int(metadata["started_ms"]), ended_ms=int(metadata["ended_ms"]), started_boot_ms=int(metadata["started_boot_ms"]), ended_boot_ms=int(metadata["ended_boot_ms"]), wall_clock_adjustment_ms=int(metadata["ended_ms"]) - int(metadata["started_ms"]) - elapsed_us / 1000, elapsed_us=elapsed_us, peak_client_inflight=peak) -def verify_entities(control: Path) -> dict: - owners = rows(control / "entity-owners.tsv") +def verify_capacity_windows(control: Path, positions: dict[int, list[int]]) -> list[dict]: + schedule = rows(control / "capacity-windows.tsv") + assert schedule, "missing capacity schedule" + assert [int(row["window_id"]) for row in schedule] == list(range(len(schedule))), "missing capacity window" + windows = [] + for shape in SHAPES: + selected = [row for row in schedule if row["shape"] == shape] + assert 2 <= len(selected) <= len(CAPACITY_POINTS), f"{shape}: incomplete rate ramp" + assert [(int(row["rate_per_node"]), int(row["concurrency"])) for row in selected] == list(CAPACITY_POINTS[:len(selected)]), f"{shape}: rate ramp changed" + assert all(row["fully_served"] == "true" for row in selected[:-1]), f"{shape}: ramp continued after overload" + assert selected[-1]["fully_served"] == "false", f"{shape}: missing overload point" + for row in selected: + result = verify_window(control, 3, shape, int(row["rate_per_node"]), + int(row["concurrency"]), int(row["window_id"]), positions, + prefix="capacity") + result["fully_served_window"] = (result["fully_served_arrivals"] and + result["elapsed_us"] <= SECONDS * 1_000_000 + CAPACITY_DRAIN_GRACE_US) + assert result["fully_served_window"] == (row["fully_served"] == "true"), "mislabeled fully served rate" + windows.append(result) + assert [window["shape"] for window in windows] == [row["shape"] for row in schedule], "capacity shape order changed" + return windows + + +def verify_follower_proof(resources: dict) -> dict: + fleet_proofs = sum(resource["durability"]["response_sources"]["Fleet"] + for resource in resources.values()) + acknowledged_appends = sum(resource["durability"]["acknowledged_follower_appends"] + for resource in resources.values()) + network_appends = sum(resource["durability"]["acknowledged_network_appends"] + for resource in resources.values()) + assert fleet_proofs > 0, "follower lane returned no follower-proof responses" + assert acknowledged_appends > 0, "missing acknowledged follower append evidence" + assert network_appends > 0, "missing network follower append evidence" + for resource in resources.values(): + phases = resource["durability"]["node_log_phases"] + assert phases.get("enrolled", 0) == phases.get("active", 0) == phases.get("closed", 0) == 1, \ + "follower generation did not enroll, activate, and close exactly once" + assert resource["durability"]["node_log_epochs"] == [1], "follower epoch changed" + return dict(follower_proof_responses=fleet_proofs, follower_appends=acknowledged_appends, + network_follower_appends=network_appends) + + +def verify_root_coverage(roots: list[dict], positions: dict[int, list[int]], + identity: dict[int, tuple], cells: int) -> None: + assert [int(row["entity"]) for row in roots] == list(range(cells)) + for row in roots: + entity = int(row["entity"]) + assert (row["cell"], row["owner"], row["epoch"], row["incarnation"]) == identity[entity] + assert positions[entity], f"Cell {entity} received no acknowledged writes" + assert int(row["root_sequence"]) >= max(positions[entity]), "published root does not cover writes" + + +def verify_entities(control: Path, capacity: bool = False, follower: bool = False) -> dict: + assert not follower or capacity + stages = (3,) if capacity else STAGES + evidence_prefix = "capacity" if capacity else "entity" + owners = rows(control / f"{evidence_prefix}-owners.tsv") identity = {} - for stage in STAGES: + for stage in stages: selected = [row for row in owners if int(row["stage"]) == stage] assert [int(row["entity"]) for row in selected] == list(range(stage * CELLS_PER_NODE)) for row in selected: @@ -132,44 +344,45 @@ def verify_entities(control: Path) -> dict: assert int(row["owner"]) == entity // CELLS_PER_NODE value = (row["cell"], row["owner"], row["epoch"], row["incarnation"]) assert identity.setdefault(entity, value) == value, "existing Cell ownership changed" - ingress = list(map(int, (control / f"entity-ingress-{stage}.txt").read_text().split())) + ingress = list(map(int, (control / f"{evidence_prefix}-ingress-{stage}.txt").read_text().split())) assert len(ingress) == stage and min(ingress) > 0 and max(ingress) - min(ingress) <= 1 - assert len({value[0] for value in identity.values()}) == 80, "entity targets collapsed" + assert len({value[0] for value in identity.values()}) == stages[-1] * CELLS_PER_NODE, "entity targets collapsed" windows, positions = [], {} - for nodes in STAGES: - for shape in SHAPES: - for rate, concurrency in POINTS: - windows.append(verify_window(control, nodes, shape, rate, concurrency, len(windows), positions)) - roots = rows(control / f"entity-roots-{nodes}.tsv") - assert [int(row["entity"]) for row in roots] == list(range(nodes * CELLS_PER_NODE)) - for row in roots: - entity = int(row["entity"]) - assert (row["cell"], row["owner"], row["epoch"], row["incarnation"]) == identity[entity] - assert positions[entity], f"Cell {entity} received no acknowledged writes" - assert int(row["root_sequence"]) >= max(positions[entity]), "published root does not cover writes" + for nodes in stages: + if capacity: + windows.extend(verify_capacity_windows(control, positions)) + else: + for shape in SHAPES: + for rate, concurrency in POINTS: + windows.append(verify_window(control, nodes, shape, rate, concurrency, len(windows), positions)) + roots = rows(control / f"{evidence_prefix}-roots-{nodes}.tsv") + verify_root_coverage(roots, positions, identity, nodes * CELLS_PER_NODE) for node in range(nodes): assert sum(window["acknowledged_writes_by_node"][node] for window in windows if window["nodes"] == nodes) > 0 resources = {} - for node in range(20): + for node in range(stages[-1]): samples = rows(control / f"node-{node}-resources.tsv") assert len(samples) >= 2 assert all(int(row["active_cells"]) == CELLS_PER_NODE for row in samples) + assert all(int(row["unpublished_node_log_bytes"]) >= 0 for row in samples) for column in ("boot_ms", "cpu_usage_us", "throttled_us", "object_started", "object_finished", "bytes_read", "bytes_written"): values = [int(row[column]) for row in samples] assert values == sorted(values), f"node {node}: {column} regressed" - assert {int(row["stage"]) for row in samples} >= {stage for stage in STAGES if node < stage} + assert {int(row["stage"]) for row in samples} >= {stage for stage in stages if node < stage} local, forwarded = map(int, (control / f"node-{node}.counts").read_text().split()) assert local > 0 and forwarded > 0 observations = rows(control / f"node-{node}-objects.tsv") assert len(observations) == 99 and len({(row["operation"], row["outcome"]) for row in observations}) == 99 assert all(int(row["count"]) >= 0 for row in observations) assert any(row["operation"] == "put" and row["outcome"] == "success" and int(row["count"]) > 0 for row in observations) + object_operations = verify_object_operations(control, node, observations) waits = [int(row["object_wait_us"]) for row in rows(control / f"node-{node}-durability.tsv")] assert waits and min(waits) >= 0 resources[node] = dict(samples=len(samples), max_memory_current_bytes=max(int(row["memory_current_bytes"]) for row in samples), max_disk_file_bytes=max(int(row["disk_bytes"]) for row in samples), gateway_local=local, gateway_forwarded=forwarded, object_wait=distribution(waits), logical_object_operations=sum(int(row["count"]) for row in observations)) + resources[node]["durability"] = verify_timing_evidence(control, node, windows) for window in windows: if node >= window["nodes"]: continue @@ -183,9 +396,52 @@ def verify_entities(control: Path) -> dict: logical_object_started=int(last["object_started"]) - int(first["object_started"]), logical_object_finished=int(last["object_finished"]) - int(first["object_finished"]), max_memory_current_bytes=max(int(row["memory_current_bytes"]) for row in observed), + max_unpublished_node_log_bytes=max(int(row["unpublished_node_log_bytes"]) for row in observed), max_disk_file_bytes=max(int(row["disk_bytes"]) for row in observed), max_worker_jobs=max(int(row["worker_jobs"]) for row in observed)) + selected_objects = [row for row in object_operations + if window["started_ms"] <= int(row["at_ms"]) <= window["ended_ms"]] + window.setdefault("node_store", {})[node] = dict( + operations={operation: distribution([int(row["duration_us"]) for row in selected_objects + if row["operation"] == operation]) + for operation in sorted({row["operation"] for row in selected_objects})}, + outcomes=dict(Counter(row["outcome"] for row in selected_objects)), + requests_per_acknowledged_write=(len(selected_objects) + / max(1, window["acknowledged_writes_by_node"][node])), + requests_per_completed_action=(len(selected_objects) + / max(1, window["completed_actions_by_node"][node])), + bytes_read=sum(int(row["bytes_read"]) for row in selected_objects), + bytes_written=sum(int(row["bytes_written"]) for row in selected_objects)) + extra = {} + if capacity: + if follower: + extra.update(verify_follower_proof(resources)) + for window in windows: + root_lags = {} + for entity, acknowledged in window["latest_write_sequence_by_entity"].items(): + cell = identity[entity][0] + owner = entity // CELLS_PER_NODE + published = window["node_durability"][owner]["latest_published_sequence_by_cell"].get(cell, 0) + root_lags[entity] = max(0, acknowledged - published) + window["root_lag_commits_by_entity"] = root_lags + window["max_root_lag_commits"] = max(root_lags.values(), default=0) + capacity_curves = {} + for shape in SHAPES: + selected = [window for window in windows if window["shape"] == shape] + fully_served = selected[-2] + overloaded = selected[-1] + capacity_curves[shape] = dict( + max_fully_served_rate_per_node=fully_served["rate_per_node"], + max_fully_served_logical_writes_per_second=fully_served["completed_writes_per_second"], + first_overloaded_rate_per_node=overloaded["rate_per_node"], + first_overloaded_outcomes=overloaded["outcomes"], + max_root_lag_commits_at_fully_served_rate=fully_served["max_root_lag_commits"], + max_root_lag_commits_at_overload=overloaded["max_root_lag_commits"], + published_roots_per_second=sum( + node["published_roots_per_second"] for node in fully_served["node_durability"].values()), + ) + extra["capacity_curves"] = capacity_curves return dict(integrity_verified=True, windows=windows, resources=resources, verified_cells=len(positions), acknowledged_writes=sum(map(len, positions.values())), raw_sha256={path.name: hashlib.sha256(path.read_bytes()).hexdigest() - for path in sorted(control.glob("*.tsv"))}) + for path in sorted(control.glob("*.tsv"))}, **extra) diff --git a/crates/cellule-app/qualification/run.sh b/crates/cellule-app/qualification/run.sh index 9ca9944..76c58ed 100644 --- a/crates/cellule-app/qualification/run.sh +++ b/crates/cellule-app/qualification/run.sh @@ -8,7 +8,9 @@ case "${1:-}" in entities) role=entities; selected=entities::hosts::entity_ledgers_are_isolated_across_three_rustfs_hosts ;; entity-node) role="node-${CELLULE_PERF_PROCESS_NODE:?}"; selected=entities::process::entity_process_node ;; entity-scale) role=driver; selected=entities::process::driver::entity_process_scaling ;; - *) printf 'usage: run.sh node|driver|scale|rollout|entities|entity-node|entity-scale\n' >&2; exit 2 ;; + entity-capacity) role=driver; selected=entities::process::driver::entity_process_capacity ;; + entity-capacity-follower) role=driver; selected=entities::process::driver::entity_process_capacity_follower ;; + *) printf 'usage: run.sh node|driver|scale|rollout|entities|entity-node|entity-scale|entity-capacity|entity-capacity-follower\n' >&2; exit 2 ;; esac binary= for candidate in /target/release/deps/integration-*; do diff --git a/crates/cellule-app/qualification/scale.py b/crates/cellule-app/qualification/scale.py index 6c83f61..86dad10 100644 --- a/crates/cellule-app/qualification/scale.py +++ b/crates/cellule-app/qualification/scale.py @@ -126,7 +126,7 @@ def verify_reader_loss(control: Path, killed_node: int) -> dict: class Fleet: def __init__(self, state: Path, project: str, overrides: list[Path], workload: str = "readers"): - assert workload in ("readers", "entities") + assert workload in ("readers", "entities", "capacity", "capacity-follower") self.workload = workload self.state = state.resolve(strict=True) self.project = project @@ -136,7 +136,9 @@ def __init__(self, state: Path, project: str, overrides: list[Path], workload: s existing = self.run("docker", "ps", "-aq", "--filter", f"label=com.docker.compose.project={project}") if existing.strip(): raise ValueError("project already has containers; retain it and choose a fresh project") - self.evidence = self.state / "evidence" / ("scaling" if workload == "readers" else "entity-scaling") + evidence_name = {"readers": "scaling", "entities": "entity-scaling", "capacity": "entity-capacity", + "capacity-follower": "entity-capacity-follower"}[workload] + self.evidence = self.state / "evidence" / evidence_name self.evidence.mkdir(mode=0o1777) self.evidence.chmod(0o1777) self.control = self.evidence / "control" @@ -150,18 +152,25 @@ def __init__(self, state: Path, project: str, overrides: list[Path], workload: s node = config["services"]["node-0"] # Twenty live nodes plus one killed boot. New boots have new identities; # restarting a deterministic fixture session would bypass expiry fencing. - for index in range(3, 21): + for index in range(3, 3 if workload.startswith("capacity") else 21): added = copy.deepcopy(node) added["environment"].update( CELLULE_PERF_PROCESS_NODE=str(index), CELLULE_PERF_PROCESS_ADVERTISE=f"node-{index}:8080", ) config["services"][f"node-{index}"] = added - config["services"]["driver"]["command"] = ["scale" if workload == "readers" else "entity-scale"] - if workload == "entities": + config["services"]["driver"]["command"] = [{"readers": "scale", "entities": "entity-scale", "capacity": "entity-capacity", + "capacity-follower": "entity-capacity-follower"}[workload]] + if workload in ("entities", "capacity", "capacity-follower"): for name, service in config["services"].items(): if name.startswith("node-"): service["command"] = ["entity-node"] + if workload.startswith("capacity"): + for name, service in config["services"].items(): + if name.startswith("node-") or name == "driver": + service["environment"]["CELLULE_PERF_PROCESS_ROOT"] = f"capacity-{project}" + if workload == "capacity-follower" and name.startswith("node-"): + service["environment"]["CELLULE_PERF_PROCESS_FOLLOWER"] = "1" for service in config["services"].values(): for volume in service.get("volumes", []): if volume["target"] == "/evidence": @@ -270,9 +279,10 @@ def execute(self) -> None: self.verify() def verify(self) -> None: - assert len(self.active) == 20 and len(self.killed) == (1 if self.workload == "readers" else 0) + stages = (3,) if self.workload.startswith("capacity") else (3, 5, 10, 20) + assert len(self.active) == stages[-1] and len(self.killed) == (1 if self.workload == "readers" else 0) assert [(event["action"], event["argument"]) for event in self.events if event["action"] == "scale"] == [ - ("scale", 3), ("scale", 5), ("scale", 10), ("scale", 20) + ("scale", stage) for stage in stages ] roles = {f"node-{node}": container for node, container in self.active.items()} roles["driver"] = self.driver @@ -302,11 +312,13 @@ def verify(self) -> None: binaries.add((self.evidence / f"{role}-binary.sha256").read_text().split()[0]) assert len(binaries) == 1 source = (self.state / "evidence/source-revision.txt").read_text().strip() - if self.workload == "entities": + if self.workload in ("entities", "capacity", "capacity-follower"): result = dict(workload=self.workload, source=source, binary_sha256=binaries.pop(), - roles=reports, events=self.events, **verify_entities(self.control)) + roles=reports, events=self.events, + **verify_entities(self.control, capacity=self.workload.startswith("capacity"), + follower=self.workload == "capacity-follower")) (self.evidence / "verification.json").write_text(json.dumps(result, indent=2) + "\n") - print(f"Verified entity integrity and resources at 3/5/10/20 nodes; evidence: {self.evidence}", flush=True) + print(f"Verified entity integrity and resources at {stages} nodes; evidence: {self.evidence}", flush=True) return killed = next(iter(self.killed)) driver_log = (self.evidence / "driver.log").read_text() @@ -338,7 +350,7 @@ def main() -> None: parser.add_argument("--state", type=Path, required=True, help="prepared source, binary and evidence directory") parser.add_argument("--project", required=True, help="fresh Compose project; stopped containers are retained") parser.add_argument("--compose-file", type=Path, action="append", default=[], help="explicit image/cache override") - parser.add_argument("--workload", choices=("readers", "entities"), default="readers", help="reader replacement or scheduled writable entity traffic") + parser.add_argument("--workload", choices=("readers", "entities", "capacity", "capacity-follower"), default="readers", help="reader replacement, scaling, or fixed 12-Cell capacity traffic") args = parser.parse_args() fleet = Fleet(args.state, args.project, args.compose_file, args.workload) try: diff --git a/crates/cellule-app/qualification/test_entities.py b/crates/cellule-app/qualification/test_entities.py index 28035d2..f9b4290 100644 --- a/crates/cellule-app/qualification/test_entities.py +++ b/crates/cellule-app/qualification/test_entities.py @@ -4,8 +4,9 @@ from pathlib import Path import tempfile import unittest +from unittest.mock import patch -from entities import verify_window +from entities import destination, verify_capacity_windows, verify_follower_proof, verify_object_operations, verify_root_coverage, verify_timing_evidence, verify_window class EntityWindowEvidence(unittest.TestCase): @@ -75,5 +76,206 @@ def test_missed_arrival_is_not_fully_served(self): self.assertEqual((result["fully_served_arrivals"], result["completed_actions"]), (False, 29)) +class EntityTimingEvidence(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + (self.root / "node-0-responses.tsv").write_text( + "at_ms\tsource\tresponse_us\tconfirmation_us\n" + "100001\tFleet\t1200\t900\n100002\tObject\t2000\t1800\n") + (self.root / "node-0-executions.tsv").write_text( + "at_ms\tqueue_wait_us\tworker_round_trip_us\tsucceeded\n" + "100001\t100\t300\ttrue\n100002\t200\t400\ttrue\n") + (self.root / "node-0-publications.tsv").write_text( + "at_ms\tcell\tsequence\tqueue_wait_us\tpreparation_us\tauthority_us\ttotal_us\tsucceeded\n" + "100003\tcell-1\t1\t100\t1000\t200\t1500\ttrue\n") + (self.root / "node-0-phases.tsv").write_text( + "at_ms\tphase\telapsed_us\tsucceeded\n100003\tCapture\t700\ttrue\n") + (self.root / "node-0-captures.tsv").write_text( + "at_ms\ttotal_us\tpreparation_us\tschema_check_us\twal_read_us\tpage_collection_us\tverification_us\tencode_us\tlocal_write_us\tfsync_us\tcheckpoint_us\twal_bytes\tltx_bytes\tsucceeded\n" + "100003\t700\t100\t20\t100\t100\t50\t50\t50\t100\t50\t4096\t2048\ttrue\n") + (self.root / "node-0-publication-costs.tsv").write_text( + "at_ms\tobjects\tbytes\n100003\t2\t2048\n") + (self.root / "node-0-follower-appends.tsv").write_text( + "at_ms\tacknowledged\tbytes\n") + (self.root / "node-0-follower-network.tsv").write_text( + "at_ms\tacknowledged\tbytes\tduration_us\n") + (self.root / "node-0-node-log-events.tsv").write_text( + "at_ms\tepoch\tphase\tcovered_through\n") + self.windows = [dict(nodes=3, started_ms=100000, ended_ms=110000, elapsed_us=10_000_000)] + + def test_response_winner_and_later_publication_are_separate(self): + report = verify_timing_evidence(self.root, 0, self.windows) + self.assertEqual(report["response_sources"], dict(Fleet=1, Object=1, Recorded=0)) + self.assertEqual(self.windows[0]["node_durability"][0]["published_roots_per_second"], 0.1) + self.assertEqual(self.windows[0]["node_durability"][0]["uploaded_objects"], 2) + self.assertEqual(self.windows[0]["node_durability"][0]["actor_queue"]["count"], 2) + + def test_incomplete_timing_evidence_is_rejected(self): + (self.root / "node-0-publications.tsv").write_text( + "at_ms\tcell\tsequence\tqueue_wait_us\tpreparation_us\tauthority_us\ttotal_us\tsucceeded\n") + with self.assertRaisesRegex(AssertionError, "missing publication evidence"): + verify_timing_evidence(self.root, 0, self.windows) + + def test_missing_execution_evidence_is_rejected(self): + (self.root / "node-0-executions.tsv").write_text( + "at_ms\tqueue_wait_us\tworker_round_trip_us\tsucceeded\n") + with self.assertRaisesRegex(AssertionError, "missing command execution evidence"): + verify_timing_evidence(self.root, 0, self.windows) + + def test_duplicate_publication_is_rejected(self): + path = self.root / "node-0-publications.tsv" + lines = path.read_text().splitlines() + path.write_text("\n".join(lines + [lines[-1]]) + "\n") + with self.assertRaisesRegex(AssertionError, "duplicate publication"): + verify_timing_evidence(self.root, 0, self.windows) + + def test_active_marker_after_coverage_does_not_reset_covered_sequence(self): + path = self.root / "node-0-node-log-events.tsv" + path.write_text("at_ms\tepoch\tphase\tcovered_through\n" + "100001\t1\tenrolled\t0\n" + "100002\t1\tcoverage\t1\n" + "100003\t1\tactive\t0\n" + "100004\t1\tcoverage\t2\n" + "100005\t1\tclosed\t2\n") + report = verify_timing_evidence(self.root, 0, self.windows) + self.assertEqual(report["node_log_covered_through"], 2) + self.assertEqual(self.windows[0]["node_durability"][0]["node_log_covered_through"], 2) + + path.write_text(path.read_text().replace("100004\t1\tcoverage\t2", + "100004\t1\tcoverage\t0")) + with self.assertRaisesRegex(AssertionError, "node-log coverage regressed"): + verify_timing_evidence(self.root, 0, self.windows) + + +class CapacityScheduleEvidence(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name) + self.schedule = self.root / "capacity-windows.tsv" + self.write_schedule([ + (0, "uniform", 2, 8, "true"), (1, "uniform", 4, 16, "false"), + (2, "hot", 2, 8, "true"), (3, "hot", 4, 16, "false"), + (4, "skewed", 2, 8, "true"), (5, "skewed", 4, 16, "false"), + ]) + + def write_schedule(self, entries): + self.schedule.write_text( + "window_id\tshape\trate_per_node\tconcurrency\tfully_served\n" + + "".join("\t".join(map(str, entry)) + "\n" for entry in entries)) + + def write_complete_windows(self): + counts = [0] * 12 + sequences = [1] * 12 + for window_id, shape, rate, concurrency, _ in [ + (0, "uniform", 2, 8, True), (1, "uniform", 4, 16, False), + (2, "hot", 2, 8, True), (3, "hot", 4, 16, False), + (4, "skewed", 2, 8, True), (5, "skewed", 4, 16, False), + ]: + label = f"capacity-3-{shape}-{rate}" + started_ms = 100000 + window_id * 20000 + (self.root / f"{label}-window.tsv").write_text( + "window_id\tnodes\tshape\trate_per_node\tconcurrency\tseconds\tstarted_ms\tended_ms\tstarted_boot_ms\tended_boot_ms\telapsed_us\n" + f"{window_id}\t3\t{shape}\t{rate}\t{concurrency}\t10\t{started_ms}\t{started_ms + 10000}\t{started_ms}\t{started_ms + 10000}\t10000000\n") + samples = ["arrival\tscheduled_us\tstarted_us\telapsed_us\tentity\tkind\toutcome\tsequence\tread_sequence\tcount"] + planned = 30 * rate + for arrival in range(planned): + entity, kind = destination(shape, arrival, 12) + scheduled = arrival * 1_000_000 // (3 * rate) + if rate == 4 and arrival == planned - 1: + samples.append(f"{arrival}\t{scheduled}\t10000000\t0\t{entity}\t{kind}\tscheduler_late\t0\t0\t0") + continue + sequence = 0 + if kind == "write": + counts[entity] += 1 + sequences[entity] += 2 + sequence = sequences[entity] + samples.append(f"{arrival}\t{scheduled}\t{scheduled + 10}\t1000\t{entity}\t{kind}\tok\t{sequence}\t{sequences[entity]}\t{counts[entity]}") + (self.root / f"{label}.tsv").write_text("\n".join(samples) + "\n") + (self.root / f"{label}-readback.tsv").write_text( + "entity\texpected\tactual\tsequence\n" + + "".join(f"{entity}\t{counts[entity]}\t{counts[entity]}\t{sequences[entity]}\n" + for entity in range(12))) + + def test_complete_report_requires_all_shapes_and_overload(self): + self.write_complete_windows() + windows = verify_capacity_windows(self.root, {}) + self.assertEqual(len(windows), 6) + self.assertEqual([window["fully_served_window"] for window in windows], [True, False] * 3) + + def test_completed_arrivals_with_slow_drain_are_not_supported(self): + self.write_complete_windows() + path = self.root / "capacity-3-uniform-2-window.tsv" + path.write_text(path.read_text().replace("10000000\n", "12100000\n")) + with self.assertRaisesRegex(AssertionError, "mislabeled fully served rate"): + verify_capacity_windows(self.root, {}) + + def test_incomplete_report_is_rejected(self): + self.write_schedule([(0, "uniform", 1, 4, "true")]) + with self.assertRaisesRegex(AssertionError, "incomplete rate ramp"): + verify_capacity_windows(self.root, {}) + + def test_rate_mislabeled_as_fully_served_is_rejected(self): + self.write_schedule([ + (0, "uniform", 2, 8, "true"), (1, "uniform", 4, 16, "false"), + (2, "hot", 2, 8, "true"), (3, "hot", 4, 16, "false"), + (4, "skewed", 2, 8, "true"), (5, "skewed", 4, 16, "false"), + ]) + with patch("entities.verify_window", return_value=dict(fully_served_arrivals=False)): + with self.assertRaisesRegex(AssertionError, "mislabeled fully served rate"): + verify_capacity_windows(self.root, {}) + + +class ObjectOperationEvidence(unittest.TestCase): + def test_missing_provider_operation_is_rejected(self): + with tempfile.TemporaryDirectory() as root: + path = Path(root) / "node-0-object-operations.tsv" + path.write_text("at_ms\toperation\toutcome\tduration_us\tbytes_read\tbytes_written\n" + "100000\tput\tsuccess\t500\t0\t4096\n") + expected = [dict(operation="put", outcome="success", count="2")] + with self.assertRaisesRegex(AssertionError, "samples disagree"): + verify_object_operations(Path(root), 0, expected) + expected[0]["count"] = "1" + self.assertEqual(len(verify_object_operations(Path(root), 0, expected)), 1) + + +class FollowerProofEvidence(unittest.TestCase): + def test_follower_lane_requires_proof_and_acknowledged_append(self): + resources = {0: dict(durability=dict(response_sources=dict(Fleet=0), + acknowledged_follower_appends=1, + acknowledged_network_appends=1, + node_log_phases=dict(enrolled=1, active=1, closed=1), + node_log_epochs=[1]))} + with self.assertRaisesRegex(AssertionError, "no follower-proof responses"): + verify_follower_proof(resources) + resources[0]["durability"]["response_sources"]["Fleet"] = 1 + resources[0]["durability"]["acknowledged_follower_appends"] = 0 + with self.assertRaisesRegex(AssertionError, "missing acknowledged follower append"): + verify_follower_proof(resources) + resources[0]["durability"]["acknowledged_follower_appends"] = 2 + resources[0]["durability"]["acknowledged_network_appends"] = 0 + with self.assertRaisesRegex(AssertionError, "missing network follower append"): + verify_follower_proof(resources) + resources[0]["durability"]["acknowledged_network_appends"] = 2 + self.assertEqual(verify_follower_proof(resources), + dict(follower_proof_responses=1, follower_appends=2, + network_follower_appends=2)) + resources[0]["durability"]["node_log_phases"]["active"] = 0 + with self.assertRaisesRegex(AssertionError, "did not enroll, activate, and close"): + verify_follower_proof(resources) + + def test_root_drain_requires_every_acknowledged_sequence(self): + identity = {0: ("cell", "0", "1", "incarnation")} + positions = {0: [1, 2]} + roots = [dict(entity="0", cell="cell", owner="0", epoch="1", + incarnation="incarnation", root_sequence="1")] + with self.assertRaisesRegex(AssertionError, "published root does not cover writes"): + verify_root_coverage(roots, positions, identity, 1) + roots[0]["root_sequence"] = "2" + verify_root_coverage(roots, positions, identity, 1) + + if __name__ == "__main__": unittest.main() diff --git a/crates/cellule-app/tests/entities/process.rs b/crates/cellule-app/tests/entities/process.rs index 66db8f6..b3d1745 100644 --- a/crates/cellule-app/tests/entities/process.rs +++ b/crates/cellule-app/tests/entities/process.rs @@ -2,9 +2,11 @@ use super::super::fleet::{GatewayStats, start_gateway_peer_server}; use super::super::performance_fixture::{node_session, now_ms, rustfs_store}; +use super::super::process_follower; use super::super::process_node; use super::super::process_performance::{publish_address, wait_for_marker}; use super::*; +use cellule_runtime::follower::FollowerStore; use cellule_runtime::peer::PeerVerifier; use std::{ env, @@ -61,6 +63,7 @@ async fn entity_process_node() { .parse() .unwrap(); assert!(node < 20); + let follower_enabled = env::var("CELLULE_PERF_PROCESS_FOLLOWER").as_deref() == Ok("1"); let sync = env::var("CELLULE_PERF_PROCESS_SYNC").unwrap(); let sync = Path::new(&sync); let application = compiled_entities(); @@ -76,20 +79,38 @@ async fn entity_process_node() { let listener = TcpListener::bind(env::var("CELLULE_PERF_PROCESS_BIND").unwrap()) .await .unwrap(); + let follower_listener = if follower_enabled { + Some(TcpListener::bind("0.0.0.0:8081").await.unwrap()) + } else { + None + }; let address = tokio::net::lookup_host(env::var("CELLULE_PERF_PROCESS_ADVERTISE").unwrap()) .await .unwrap() .next() .unwrap(); let endpoint = format!("https://{address}"); - let (host, durability, readers) = process_node::start( + let (host, durability, readers) = process_node::start_configured( node, application.clone(), &layout, directory.path(), endpoint.clone(), + follower_enabled, ) .await; + let follower_server = follower_listener.map(|listener| { + let store = host + .owned_component::(cellule_host::FOLLOWER_STORE_COMPONENT) + .unwrap(); + process_follower::serve( + listener, + cellule_runtime::identity::NodeId::from_bytes(*node_session(node).as_bytes()), + (*store).clone(), + process_node::directory(&layout, ®istry), + process_node::signing_key(node), + ) + }); let mut handles = Vec::new(); for entity in node * ENTITIES_PER_NODE..(node + 1) * ENTITIES_PER_NODE { handles.push( @@ -129,6 +150,16 @@ async fn entity_process_node() { )), ); publish_address(&sync.join(format!("node-{node}.ready")), address); + if follower_enabled { + let deadline = Instant::now() + Duration::from_secs(30); + while host.runtime().node_durability().is_none() { + assert!( + Instant::now() < deadline, + "follower enrollment did not complete" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + } publish_marker(&sync.join(format!("node-{node}.serving")), []); let mut observations = observation::NodeObservations::new(sync, node); let mut next_sample = Instant::now(); @@ -164,7 +195,7 @@ async fn entity_process_node() { observations.sample(stage, &host, directory.path(), &storage, &stats); assert_eq!(host.stats().active_cells(), ENTITIES_PER_NODE); host.shutdown().await.unwrap(); - observations.finish(&storage, &durability.object_waits()); + observations.finish(&storage, &durability); let (local, forwarded) = stats.counts(); assert!(local > 0 && forwarded > 0); publish_marker( @@ -174,4 +205,13 @@ async fn entity_process_node() { publish_marker(&sync.join(format!("node-{node}.done")), []); server.abort(); let _ = server.await; + if let Some(server) = follower_server { + // Other owners can still be retiring their lanes during concurrent + // shutdown; keep every follower listener available until all drain. + for peer in 0..3 { + wait_for_marker(&sync.join(format!("node-{peer}.done"))).await; + } + server.abort(); + let _ = server.await; + } } diff --git a/crates/cellule-app/tests/entities/process/driver.rs b/crates/cellule-app/tests/entities/process/driver.rs index 1949beb..dba700e 100644 --- a/crates/cellule-app/tests/entities/process/driver.rs +++ b/crates/cellule-app/tests/entities/process/driver.rs @@ -14,9 +14,11 @@ use std::{ use tokio::task::JoinSet; const WINDOW_SECONDS: usize = 10; +const CAPACITY_DRAIN_GRACE_US: u64 = 2_000_000; struct Window { id: usize, + prefix: &'static str, nodes: usize, shape: &'static str, rate_per_node: usize, @@ -26,8 +28,8 @@ struct Window { impl Window { fn label(&self) -> String { format!( - "entities-{}-{}-{}", - self.nodes, self.shape, self.rate_per_node + "{}-{}-{}-{}", + self.prefix, self.nodes, self.shape, self.rate_per_node ) } } @@ -48,6 +50,22 @@ struct Sample { #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "Compose controller required for scheduled entity traffic on 3/5/10/20 nodes"] async fn entity_process_scaling() { + run_entity_process(false, false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "Compose controller required for fixed-Cell scheduled capacity traffic"] +async fn entity_process_capacity() { + run_entity_process(true, false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "Compose controller required for networked follower-proof capacity traffic"] +async fn entity_process_capacity_follower() { + run_entity_process(true, true).await; +} + +async fn run_entity_process(capacity: bool, follower_enabled: bool) { let sync = env::var("CELLULE_PERF_PROCESS_SYNC").unwrap(); let sync = Path::new(&sync); let mut controller = Controller::new(sync); @@ -58,11 +76,24 @@ async fn entity_process_scaling() { *ApplicationId::from_bytes([82; 16]).as_bytes(), ); let authority = CellAuthority::new(layout.clone()); - let mut owners = BufWriter::new(File::create(sync.join("entity-owners.tsv")).unwrap()); + let evidence_prefix = if capacity { "capacity" } else { "entity" }; + let window_prefix = if capacity { "capacity" } else { "entities" }; + let stages: &[usize] = if capacity { &[3] } else { &[3, 5, 10, 20] }; + let mut owners = + BufWriter::new(File::create(sync.join(format!("{evidence_prefix}-owners.tsv"))).unwrap()); writeln!(owners, "stage\tentity\tcell\towner\tepoch\tincarnation").unwrap(); let mut expected = Vec::new(); let mut window_id = 0; - for nodes in [3, 5, 10, 20] { + let mut capacity_windows = capacity.then(|| { + let mut output = BufWriter::new(File::create(sync.join("capacity-windows.tsv")).unwrap()); + writeln!( + output, + "window_id\tshape\trate_per_node\tconcurrency\tfully_served" + ) + .unwrap(); + output + }); + for &nodes in stages { assert_eq!( controller.command("scale", nodes).await, (0..nodes).collect::>() @@ -70,6 +101,9 @@ async fn entity_process_scaling() { for node in 0..nodes { wait_for_marker(&sync.join(format!("node-{node}.serving"))).await; } + if follower_enabled { + assert_eq!(nodes, 3); + } let addresses = endpoints(sync, nodes).await; publish_marker(&sync.join("entity-stage.request"), nodes.to_string()); for node in 0..nodes { @@ -114,20 +148,59 @@ async fn entity_process_scaling() { } owners.flush().unwrap(); for shape in ["uniform", "hot", "skewed"] { - for (rate_per_node, concurrency) in [(1, 4), (4, 16), (16, 64)] { + let mut served = false; + let mut overloaded = false; + let points: &[(usize, usize)] = if capacity { + &[ + (2, 8), + (4, 16), + (16, 64), + (24, 96), + (32, 128), + (48, 192), + (64, 256), + (96, 256), + (128, 256), + (192, 256), + (256, 256), + (1024, 256), + ] + } else { + &[(1, 4), (4, 16), (16, 64)] + }; + for &(rate_per_node, concurrency) in points { let window = Window { id: window_id, + prefix: window_prefix, nodes, shape, rate_per_node, concurrency, }; - run_window(sync, &window, client.clone(), &mut expected).await; + let fully_served = run_window(sync, &window, client.clone(), &mut expected).await; + if let Some(output) = capacity_windows.as_mut() { + writeln!( + output, + "{window_id}\t{shape}\t{rate_per_node}\t{concurrency}\t{fully_served}" + ) + .unwrap(); + output.flush().unwrap(); + } window_id += 1; + served |= fully_served; + if capacity && !fully_served { + overloaded = true; + break; + } + } + if capacity { + assert!(served, "{shape}: no fully served capacity point"); + assert!(overloaded, "{shape}: rate ramp did not reach overload"); } } - let mut roots = - BufWriter::new(File::create(sync.join(format!("entity-roots-{nodes}.tsv"))).unwrap()); + let mut roots = BufWriter::new( + File::create(sync.join(format!("{evidence_prefix}-roots-{nodes}.tsv"))).unwrap(), + ); writeln!( roots, "entity\tcell\towner\tepoch\tincarnation\troot_sequence\troot_digest" @@ -156,7 +229,7 @@ async fn entity_process_scaling() { assert!(counts.iter().all(|count| *count > 0)); assert!(counts.iter().max().unwrap() - counts.iter().min().unwrap() <= 1); publish_marker( - &sync.join(format!("entity-ingress-{nodes}.txt")), + &sync.join(format!("{evidence_prefix}-ingress-{nodes}.txt")), counts .iter() .map(usize::to_string) @@ -171,7 +244,7 @@ async fn entity_process_scaling() { let _ = server.await; } publish_marker(&sync.join("stop"), []); - for node in 0..20 { + for node in 0..stages[stages.len() - 1] { wait_for_marker(&sync.join(format!("node-{node}.done"))).await; } assert!( @@ -210,7 +283,7 @@ async fn run_window( window: &Window, client: Arc, expected: &mut [u64], -) { +) -> bool { let rate = window.nodes * window.rate_per_node; let planned = rate * WINDOW_SECONDS; let label = window.label(); @@ -308,6 +381,9 @@ async fn run_window( println!( "ENTITY_WINDOW label={label} planned={planned} complete={complete} elapsed_us={elapsed_us}" ); + complete == planned + && (window.prefix != "capacity" + || elapsed_us <= WINDOW_SECONDS as u64 * 1_000_000 + CAPACITY_DRAIN_GRACE_US) } fn retain_sample(output: &mut BufWriter, samples: &mut Vec, sample: Sample) { diff --git a/crates/cellule-app/tests/entities/process/observation.rs b/crates/cellule-app/tests/entities/process/observation.rs index c47378c..8767ca2 100644 --- a/crates/cellule-app/tests/entities/process/observation.rs +++ b/crates/cellule-app/tests/entities/process/observation.rs @@ -1,11 +1,15 @@ //! Cumulative provider operations and per-node Linux resource samples. use super::*; +use crate::performance_fixture::DurabilityRecorder; use cellule_store::{StorageObservation, StorageObserver, StorageOperation, StorageOutcome}; use std::{ fs::File, io::{BufWriter, Write}, - sync::atomic::{AtomicU64, Ordering}, + sync::{ + Mutex, + atomic::{AtomicU64, Ordering}, + }, }; #[derive(Default)] @@ -15,6 +19,7 @@ pub(super) struct StorageCounters { outcomes: [[AtomicU64; 9]; 11], bytes_read: AtomicU64, bytes_written: AtomicU64, + samples: Mutex>, } impl StorageObserver for StorageCounters { @@ -30,13 +35,24 @@ impl StorageObserver for StorageCounters { self.bytes_written .fetch_add(observation.bytes_written, Ordering::Relaxed); self.finished.fetch_add(1, Ordering::Relaxed); + self.samples.lock().unwrap().push((now_ms(), observation)); } } pub(super) struct NodeObservations { resources: BufWriter, objects: BufWriter, + object_operations: BufWriter, durability: BufWriter, + responses: BufWriter, + executions: BufWriter, + publications: BufWriter, + phases: BufWriter, + captures: BufWriter, + publication_costs: BufWriter, + follower_appends: BufWriter, + follower_network: BufWriter, + node_log_events: BufWriter, } impl NodeObservations { @@ -45,11 +61,21 @@ impl NodeObservations { BufWriter::new(File::create(sync.join(format!("node-{node}-{name}.tsv"))).unwrap()) }; let mut resources = create("resources"); - writeln!(resources, "at_ms\tboot_ms\tstage\tactive_cells\tworker_jobs\tprimitive_jobs\thydration_jobs\tretained_bytes\tdisk_reserved_bytes\tdisk_bytes\tcpu_usage_us\tthrottled_us\tmemory_current_bytes\tmemory_peak_bytes\tgateway_local\tgateway_forwarded\tobject_started\tobject_finished\tbytes_read\tbytes_written").unwrap(); + writeln!(resources, "at_ms\tboot_ms\tstage\tactive_cells\tworker_jobs\tprimitive_jobs\thydration_jobs\tretained_bytes\tunpublished_node_log_bytes\tdisk_reserved_bytes\tdisk_bytes\tcpu_usage_us\tthrottled_us\tmemory_current_bytes\tmemory_peak_bytes\tgateway_local\tgateway_forwarded\tobject_started\tobject_finished\tbytes_read\tbytes_written").unwrap(); Self { resources, objects: create("objects"), + object_operations: create("object-operations"), durability: create("durability"), + responses: create("responses"), + executions: create("executions"), + publications: create("publications"), + phases: create("phases"), + captures: create("captures"), + publication_costs: create("publication-costs"), + follower_appends: create("follower-appends"), + follower_network: create("follower-network"), + node_log_events: create("node-log-events"), } } @@ -88,6 +114,7 @@ impl NodeObservations { stats.primitive_jobs() as u64, stats.hydration_jobs() as u64, stats.retained_bytes() as u64, + stats.unpublished_node_log_bytes(), stats.local_disk_reserved_bytes(), disk_bytes(root), cpu_value("usage_usec"), @@ -110,7 +137,7 @@ impl NodeObservations { self.resources.flush().unwrap(); } - pub(super) fn finish(&mut self, storage: &StorageCounters, waits: &[Duration]) { + pub(super) fn finish(&mut self, storage: &StorageCounters, durability: &DurabilityRecorder) { writeln!(self.objects, "operation\toutcome\tcount").unwrap(); for operation in StorageOperation::ALL { for outcome in StorageOutcome::ALL { @@ -125,13 +152,137 @@ impl NodeObservations { .unwrap(); } } + writeln!( + self.object_operations, + "at_ms\toperation\toutcome\tduration_us\tbytes_read\tbytes_written" + ) + .unwrap(); + for (at_ms, observation) in storage.samples.lock().unwrap().iter() { + writeln!( + self.object_operations, + "{at_ms}\t{}\t{}\t{}\t{}\t{}", + observation.operation.label(), + observation.outcome.label(), + observation.duration.as_micros(), + observation.bytes_read, + observation.bytes_written + ) + .unwrap(); + } writeln!(self.durability, "object_wait_us").unwrap(); + let waits = durability.object_waits(); assert!(!waits.is_empty()); - for wait in waits { + for wait in &waits { writeln!(self.durability, "{}", wait.as_micros()).unwrap(); } + writeln!( + self.responses, + "at_ms\tsource\tresponse_us\tconfirmation_us" + ) + .unwrap(); + for (at_ms, source, elapsed, confirmation) in durability.responses() { + writeln!( + self.responses, + "{at_ms}\t{source:?}\t{}\t{}", + elapsed.as_micros(), + confirmation.as_micros() + ) + .unwrap(); + } + writeln!( + self.executions, + "at_ms\tqueue_wait_us\tworker_round_trip_us\tsucceeded" + ) + .unwrap(); + for (at_ms, queue_wait, worker_round_trip, succeeded) in durability.executions() { + writeln!( + self.executions, + "{at_ms}\t{}\t{}\t{succeeded}", + queue_wait.as_micros(), + worker_round_trip.as_micros() + ) + .unwrap(); + } + writeln!(self.publications, "at_ms\tcell\tsequence\tqueue_wait_us\tpreparation_us\tauthority_us\ttotal_us\tsucceeded").unwrap(); + for (at_ms, cell, timing) in durability.publications() { + writeln!( + self.publications, + "{at_ms}\t{cell:?}\t{}\t{}\t{}\t{}\t{}\t{}", + timing.commit_sequence, + timing.queue_wait.as_micros(), + timing.preparation.as_micros(), + timing.authority.as_micros(), + timing.total.as_micros(), + timing.succeeded + ) + .unwrap(); + } + writeln!(self.phases, "at_ms\tphase\telapsed_us\tsucceeded").unwrap(); + for (at_ms, phase, elapsed, succeeded) in durability.phases() { + writeln!( + self.phases, + "{at_ms}\t{phase:?}\t{}\t{succeeded}", + elapsed.as_micros() + ) + .unwrap(); + } + writeln!(self.captures, "at_ms\ttotal_us\tpreparation_us\tschema_check_us\twal_read_us\tpage_collection_us\tverification_us\tencode_us\tlocal_write_us\tfsync_us\tcheckpoint_us\twal_bytes\tltx_bytes\tsucceeded").unwrap(); + for (at_ms, timing, succeeded) in durability.captures() { + writeln!( + self.captures, + "{at_ms}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{succeeded}", + timing.total_nanos / 1000, + timing.preparation_nanos / 1000, + timing.schema_check_nanos / 1000, + timing.wal_read_nanos / 1000, + timing.page_collection_nanos / 1000, + timing.verification_nanos / 1000, + timing.encode_nanos / 1000, + timing.local_write_nanos / 1000, + timing.fsync_nanos / 1000, + timing.checkpoint_nanos / 1000, + timing.wal_bytes, + timing.ltx_bytes + ) + .unwrap(); + } + writeln!(self.publication_costs, "at_ms\tobjects\tbytes").unwrap(); + for (at_ms, objects, bytes) in durability.publication_costs() { + writeln!(self.publication_costs, "{at_ms}\t{objects}\t{bytes}").unwrap(); + } + writeln!(self.follower_appends, "at_ms\tacknowledged\tbytes").unwrap(); + for (at_ms, acknowledged, bytes) in durability.follower_appends() { + writeln!(self.follower_appends, "{at_ms}\t{acknowledged}\t{bytes}").unwrap(); + } + writeln!( + self.follower_network, + "at_ms\tacknowledged\tbytes\tduration_us" + ) + .unwrap(); + for (at_ms, acknowledged, bytes, elapsed) in durability.follower_network() { + writeln!( + self.follower_network, + "{at_ms}\t{acknowledged}\t{bytes}\t{}", + elapsed.as_micros() + ) + .unwrap(); + } + writeln!(self.node_log_events, "at_ms\tepoch\tphase\tcovered_through").unwrap(); + for (at_ms, epoch, phase, through) in durability.node_log_events() { + writeln!(self.node_log_events, "{at_ms}\t{epoch}\t{phase}\t{through}").unwrap(); + } self.objects.flush().unwrap(); + self.object_operations.flush().unwrap(); self.durability.flush().unwrap(); + self.responses.flush().unwrap(); + self.executions.flush().unwrap(); + self.publications.flush().unwrap(); + self.phases.flush().unwrap(); + self.captures.flush().unwrap(); + self.publication_costs.flush().unwrap(); + self.follower_appends.flush().unwrap(); + self.follower_network.flush().unwrap(); + self.node_log_events.flush().unwrap(); } } diff --git a/crates/cellule-app/tests/integration.rs b/crates/cellule-app/tests/integration.rs index a9de1be..97044ee 100644 --- a/crates/cellule-app/tests/integration.rs +++ b/crates/cellule-app/tests/integration.rs @@ -74,6 +74,7 @@ mod host; mod performance; mod performance_fixture; mod primitives; +mod process_follower; mod process_node; mod process_performance; mod process_recruitment; diff --git a/crates/cellule-app/tests/performance_fixture.rs b/crates/cellule-app/tests/performance_fixture.rs index 4e1f7c7..1ef98cb 100644 --- a/crates/cellule-app/tests/performance_fixture.rs +++ b/crates/cellule-app/tests/performance_fixture.rs @@ -8,7 +8,8 @@ use std::{ }; use cellule_host::{CellNode, CellNodeBuilder}; -use cellule_runtime::fleet::telemetry::CellTelemetry; +use cellule_ltx::{CaptureTiming, LtxPhase}; +use cellule_runtime::fleet::telemetry::{CellTelemetry, CommandResponseSource, PublicationTiming}; use cellule_runtime::node::lease::NodeLeaseGuard; use cellule_runtime::node::log::DurabilitySource; use cellule_runtime::peer::{ @@ -144,22 +145,147 @@ pub(super) struct PerfFixture { } #[derive(Default)] -pub(super) struct DurabilityRecorder(Mutex>); +pub(super) struct DurabilityRecorder { + proofs: Mutex>, + responses: Mutex>, + executions: Mutex>, + publications: Mutex>, + phases: Mutex>, + captures: Mutex>, + publication_costs: Mutex>, + follower_appends: Mutex>, + follower_network: Mutex>, + node_log_events: Mutex>, +} impl DurabilityRecorder { pub(super) fn object_waits(&self) -> Vec { - self.0 + self.proofs .lock() .unwrap() .iter() .filter_map(|(source, waited)| (*source == DurabilitySource::Object).then_some(*waited)) .collect() } + + pub(super) fn responses(&self) -> Vec<(i64, CommandResponseSource, Duration, Duration)> { + self.responses.lock().unwrap().clone() + } + + pub(super) fn executions(&self) -> Vec<(i64, Duration, Duration, bool)> { + self.executions.lock().unwrap().clone() + } + + pub(super) fn publications(&self) -> Vec<(i64, cellule_runtime::CellId, PublicationTiming)> { + self.publications.lock().unwrap().clone() + } + + pub(super) fn phases(&self) -> Vec<(i64, LtxPhase, Duration, bool)> { + self.phases.lock().unwrap().clone() + } + + pub(super) fn captures(&self) -> Vec<(i64, CaptureTiming, bool)> { + self.captures.lock().unwrap().clone() + } + + pub(super) fn publication_costs(&self) -> Vec<(i64, u64, u64)> { + self.publication_costs.lock().unwrap().clone() + } + + pub(super) fn follower_appends(&self) -> Vec<(i64, bool, u64)> { + self.follower_appends.lock().unwrap().clone() + } + + pub(super) fn record_follower_network( + &self, + acknowledged: bool, + bytes: u64, + elapsed: Duration, + ) { + self.follower_network + .lock() + .unwrap() + .push((now_ms(), acknowledged, bytes, elapsed)); + } + + pub(super) fn follower_network(&self) -> Vec<(i64, bool, u64, Duration)> { + self.follower_network.lock().unwrap().clone() + } + + pub(super) fn record_node_log_event(&self, epoch: u64, phase: &'static str, through: u64) { + self.node_log_events + .lock() + .unwrap() + .push((now_ms(), epoch, phase, through)); + } + + pub(super) fn node_log_events(&self) -> Vec<(i64, u64, &'static str, u64)> { + self.node_log_events.lock().unwrap().clone() + } } impl CellTelemetry for DurabilityRecorder { fn durability_proof(&self, source: DurabilitySource, waited: Duration) { - self.0.lock().unwrap().push((source, waited)); + self.proofs.lock().unwrap().push((source, waited)); + } + + fn command_response( + &self, + source: CommandResponseSource, + elapsed: Duration, + confirmation: Duration, + ) { + self.responses + .lock() + .unwrap() + .push((now_ms(), source, elapsed, confirmation)); + } + + fn command_execution( + &self, + queue_wait: Duration, + worker_round_trip: Duration, + succeeded: bool, + ) { + self.executions + .lock() + .unwrap() + .push((now_ms(), queue_wait, worker_round_trip, succeeded)); + } + + fn publication_completed(&self, cell: cellule_runtime::CellId, timing: PublicationTiming) { + self.publications + .lock() + .unwrap() + .push((now_ms(), cell, timing)); + } + + fn ltx_phase(&self, phase: LtxPhase, elapsed: Duration, succeeded: bool) { + self.phases + .lock() + .unwrap() + .push((now_ms(), phase, elapsed, succeeded)); + } + + fn ltx_capture(&self, timing: &CaptureTiming, succeeded: bool) { + self.captures + .lock() + .unwrap() + .push((now_ms(), *timing, succeeded)); + } + + fn publication_cost(&self, objects: u64, bytes: u64) { + self.publication_costs + .lock() + .unwrap() + .push((now_ms(), objects, bytes)); + } + + fn node_log_append(&self, acknowledged: bool, bytes: u64) { + self.follower_appends + .lock() + .unwrap() + .push((now_ms(), acknowledged, bytes)); } } diff --git a/crates/cellule-app/tests/process_follower.rs b/crates/cellule-app/tests/process_follower.rs new file mode 100644 index 0000000..753beb9 --- /dev/null +++ b/crates/cellule-app/tests/process_follower.rs @@ -0,0 +1,1224 @@ +//! Test-only authenticated network transport for private-disk follower lanes. + +use super::performance_fixture::{DurabilityRecorder, now_ms}; +use bytes::Bytes; +use cellule_host::{FacilityResult, NodeDurabilityProvider}; +use cellule_ltx::Limits; +use cellule_runtime::fleet::telemetry::CellTelemetryHandle; +use cellule_runtime::follower::{FollowerReceipt, FollowerStore, FollowerTailPage}; +use cellule_runtime::identity::NodeId; +use cellule_runtime::node::durability::{NodeDurabilityConfig, NodeLogAuthority}; +use cellule_runtime::node::lease::NodeLeaseGuard; +use cellule_runtime::node::log::NodeLogRotationBarrier; +use cellule_runtime::node::log_transport::{ + AppendRequest, NodeLogTransport, RetireRequest, SealRequest, TailRequest, +}; +use cellule_runtime::node::{NodeAdvertisement, NodeDirectory, VersionedNodeAdvertisement}; +use cellule_runtime::{Error, Result, SessionId}; +use ed25519_dalek::{Signature, Signer, SigningKey, Verifier}; +use futures_util::future::BoxFuture; +use prost::Message; +use std::{ + collections::HashMap, + future::Future, + net::SocketAddr, + pin::Pin, + sync::{Arc, Mutex as StdMutex}, + time::{Duration, Instant}, +}; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::{Mutex, Semaphore}, +}; + +const REQUEST_DOMAIN: &[u8] = b"cellule.test-follower.request.v1\0"; +const RESPONSE_DOMAIN: &[u8] = b"cellule.test-follower.response.v1\0"; +const MAX_REQUEST_BYTES: usize = 8 << 20; +const MAX_RESPONSE_BYTES: usize = 2 << 20; +const REQUEST_TIMEOUT: Duration = Duration::from_secs(10); +const MAX_DEADLINE_AHEAD_MS: i64 = 10_000; +const MAX_APPEND_FRAMES: usize = 64; + +#[derive(Clone, PartialEq, Message)] +struct SignedWire { + #[prost(bytes = "vec", tag = "1")] + body: Vec, + #[prost(bytes = "vec", tag = "2")] + signature: Vec, +} + +#[derive(Clone, PartialEq, Message)] +struct RequestWire { + #[prost(bytes = "vec", tag = "1")] + sender: Vec, + #[prost(bytes = "vec", tag = "2")] + member: Vec, + #[prost(bytes = "vec", tag = "3")] + leader: Vec, + #[prost(uint64, tag = "4")] + epoch: u64, + #[prost(uint32, tag = "5")] + operation: u32, + #[prost(bytes = "vec", repeated, tag = "6")] + frames: Vec>, + #[prost(uint64, tag = "7")] + covered_through: u64, + #[prost(uint64, tag = "8")] + first_sequence: u64, + #[prost(int64, tag = "9")] + deadline_ms: i64, +} + +#[derive(Clone, PartialEq, Message)] +struct ResponseWire { + #[prost(bytes = "vec", tag = "1")] + member: Vec, + #[prost(bytes = "vec", tag = "2")] + request_digest: Vec, + #[prost(uint32, tag = "3")] + status: u32, + #[prost(uint64, tag = "4")] + base_sequence: u64, + #[prost(uint64, tag = "5")] + durable_through: u64, + #[prost(bytes = "vec", repeated, tag = "6")] + frames: Vec>, + #[prost(uint64, optional, tag = "7")] + next_sequence: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Operation { + Append = 1, + Seal = 2, + Retire = 3, + TailPage = 4, +} + +impl TryFrom for Operation { + type Error = Error; + + fn try_from(value: u32) -> Result { + match value { + 1 => Ok(Self::Append), + 2 => Ok(Self::Seal), + 3 => Ok(Self::Retire), + 4 => Ok(Self::TailPage), + _ => Err(Error::PeerAuthorization("unsupported follower operation")), + } + } +} + +fn session(bytes: &[u8]) -> Result { + let bytes: [u8; 16] = bytes + .try_into() + .map_err(|_| Error::Peer("invalid follower session"))?; + Ok(SessionId::from_bytes(bytes)) +} + +fn node(bytes: &[u8]) -> Result { + let bytes: [u8; 16] = bytes + .try_into() + .map_err(|_| Error::Peer("invalid follower node"))?; + Ok(NodeId::from_bytes(bytes)) +} + +fn validate_request(request: &RequestWire, member: NodeId, now: i64) -> Result { + let operation = Operation::try_from(request.operation)?; + if node(&request.member)? != member + || request.epoch == 0 + || request.deadline_ms <= now + || request.deadline_ms > now.saturating_add(MAX_DEADLINE_AHEAD_MS) + { + return Err(Error::PeerAuthorization( + "follower request scope or deadline is invalid", + )); + } + if operation == Operation::Append + && (request.frames.is_empty() || request.frames.len() > MAX_APPEND_FRAMES) + { + return Err(Error::PeerAuthorization("invalid follower append batch")); + } + if operation != Operation::Append && !request.frames.is_empty() { + return Err(Error::PeerAuthorization("unexpected follower frames")); + } + Ok(operation) +} + +fn signed(body: Vec, key: &SigningKey, domain: &[u8]) -> Vec { + let mut message = Vec::with_capacity(domain.len() + body.len()); + message.extend_from_slice(domain); + message.extend_from_slice(&body); + SignedWire { + body, + signature: key.sign(&message).to_bytes().to_vec(), + } + .encode_to_vec() +} + +fn verify(wire: &[u8], key: ed25519_dalek::VerifyingKey, domain: &[u8]) -> Result> { + let signed = SignedWire::decode(wire).map_err(|_| Error::Peer("invalid follower envelope"))?; + let signature = Signature::from_slice(&signed.signature) + .map_err(|_| Error::PeerAuthorization("invalid follower signature"))?; + let mut message = Vec::with_capacity(domain.len() + signed.body.len()); + message.extend_from_slice(domain); + message.extend_from_slice(&signed.body); + key.verify(&message, &signature) + .map_err(|_| Error::PeerAuthorization("follower signature does not match enrollment"))?; + Ok(signed.body) +} + +async fn follower_address(node: &NodeAdvertisement) -> Result { + let endpoint = node + .endpoint() + .strip_prefix("https://") + .ok_or(Error::Peer("follower endpoint is invalid"))?; + let (host, gateway_port) = endpoint + .rsplit_once(':') + .ok_or(Error::Peer("follower endpoint has no gateway port"))?; + let follower_port = gateway_port + .parse::() + .ok() + .and_then(|port| port.checked_add(1)) + .ok_or(Error::Peer("follower endpoint port is invalid"))?; + tokio::net::lookup_host(format!("{host}:{follower_port}")) + .await + .map_err(transport_io)? + .next() + .ok_or(Error::Peer("follower endpoint has no socket address")) +} + +fn transport_io(source: std::io::Error) -> Error { + Error::PeerTransportUnknown { + context: "follower fixture round trip", + source: Box::new(source), + } +} + +async fn receive(socket: &mut TcpStream, maximum: usize) -> Result> { + let mut length = [0; 4]; + socket.read_exact(&mut length).await.map_err(transport_io)?; + let length = u32::from_be_bytes(length) as usize; + if length == 0 || length > maximum { + return Err(Error::Peer("follower message exceeds byte limit")); + } + let mut bytes = vec![0; length]; + socket.read_exact(&mut bytes).await.map_err(transport_io)?; + Ok(bytes) +} + +async fn send(socket: &mut TcpStream, bytes: &[u8], maximum: usize) -> Result<()> { + if bytes.is_empty() || bytes.len() > maximum { + return Err(Error::Peer("follower message exceeds byte limit")); + } + let length = + u32::try_from(bytes.len()).map_err(|_| Error::Peer("follower message too large"))?; + socket + .write_all(&length.to_be_bytes()) + .await + .map_err(transport_io)?; + socket.write_all(bytes).await.map_err(transport_io)?; + Ok(()) +} + +/// One test-only network transport. Every reply is signed by its enrolled member. +pub(super) struct ProcessFollowerTransport { + session: SessionId, + key: SigningKey, + directory: NodeDirectory, + members: StdMutex>, + observation: Option>, +} + +struct CachedMember { + pinned_session: SessionId, + pinned_key: [u8; 32], + pinned_endpoint: String, + advertisement: NodeAdvertisement, + observed_at: Instant, +} + +impl ProcessFollowerTransport { + pub(super) fn new( + session: SessionId, + key: SigningKey, + directory: NodeDirectory, + members: HashMap, + observation: Option>, + ) -> Result { + Ok(Self { + session, + key, + directory, + observation, + members: StdMutex::new( + members + .into_iter() + .map(|(member, advertisement)| { + Ok(( + member, + CachedMember { + pinned_session: advertisement.session(), + pinned_key: advertisement.verifying_key()?.to_bytes(), + pinned_endpoint: advertisement.endpoint().to_owned(), + advertisement, + observed_at: Instant::now(), + }, + )) + }) + .collect::>>()?, + ), + }) + } + + async fn member(&self, member: NodeId) -> Result { + let (current, pinned_session, pinned_key, pinned_endpoint, observed_at) = { + let members = self + .members + .lock() + .map_err(|_| Error::Peer("follower member cache lock poisoned"))?; + let cached = members + .get(&member) + .ok_or(Error::PeerAuthorization("follower member was not enrolled"))?; + ( + cached.advertisement.clone(), + cached.pinned_session, + cached.pinned_key, + cached.pinned_endpoint.clone(), + cached.observed_at, + ) + }; + let now = now_ms(); + if observed_at.elapsed() < Duration::from_secs(1) + && current.expires_at_ms() > now.saturating_add(1_000) + { + return Ok(current); + } + let fresh = self + .directory + .resolve_node(member, now) + .await? + .ok_or(Error::Fenced)?; + if fresh.session() != pinned_session + || fresh.verifying_key()?.to_bytes() != pinned_key + || fresh.endpoint() != pinned_endpoint + { + return Err(Error::Fenced); + } + let mut members = self + .members + .lock() + .map_err(|_| Error::Peer("follower member cache lock poisoned"))?; + let cached = members + .get_mut(&member) + .ok_or(Error::PeerAuthorization("follower member was not enrolled"))?; + cached.advertisement = fresh.clone(); + cached.observed_at = Instant::now(); + Ok(fresh) + } + + async fn round_trip(&self, member: NodeId, mut request: RequestWire) -> Result { + let append = request.operation == Operation::Append as u32; + let append_bytes = request.frames.iter().map(Vec::len).sum::(); + let started = Instant::now(); + let result = self.round_trip_inner(member, &mut request).await; + if append && let Some(observation) = &self.observation { + observation.record_follower_network( + result.is_ok(), + u64::try_from(append_bytes).unwrap_or(u64::MAX), + started.elapsed(), + ); + } + result + } + + async fn round_trip_inner( + &self, + member: NodeId, + request: &mut RequestWire, + ) -> Result { + let enrolled = self.member(member).await?; + if enrolled.node() != member || enrolled.expires_at_ms() <= now_ms() { + return Err(Error::Fenced); + } + request.sender = self.session.as_bytes().to_vec(); + request.member = member.as_bytes().to_vec(); + request.deadline_ms = now_ms() + .checked_add(MAX_DEADLINE_AHEAD_MS) + .ok_or(Error::Deadline)?; + let body = request.encode_to_vec(); + let request_digest = blake3::hash(&body); + let encoded = signed(body, &self.key, REQUEST_DOMAIN); + if encoded.len() > MAX_REQUEST_BYTES { + return Err(Error::Peer("follower request exceeds byte limit")); + } + let address = follower_address(&enrolled).await?; + let response = tokio::time::timeout(REQUEST_TIMEOUT, async { + let mut socket = TcpStream::connect(address).await.map_err(transport_io)?; + send(&mut socket, &encoded, MAX_REQUEST_BYTES).await?; + receive(&mut socket, MAX_RESPONSE_BYTES).await + }) + .await + .map_err(|source| Error::PeerTransportUnknown { + context: "follower fixture deadline", + source: Box::new(source), + })??; + let body = verify(&response, enrolled.verifying_key()?, RESPONSE_DOMAIN)?; + let response = ResponseWire::decode(body.as_slice()) + .map_err(|_| Error::Peer("invalid follower response"))?; + if response.member != member.as_bytes() + || response.request_digest != request_digest.as_bytes() + { + return Err(Error::PeerAuthorization( + "follower response was not bound to request", + )); + } + match response.status { + 0 => Ok(response), + 1 => Err(Error::PeerAuthorization("follower request was refused")), + _ => Err(Error::Peer("follower operation failed")), + } + } +} + +fn request(operation: Operation, leader: SessionId, epoch: u64) -> RequestWire { + RequestWire { + sender: Vec::new(), + member: Vec::new(), + leader: leader.as_bytes().to_vec(), + epoch, + operation: operation as u32, + frames: Vec::new(), + covered_through: 0, + first_sequence: 0, + deadline_ms: 0, + } +} + +impl NodeLogTransport for ProcessFollowerTransport { + fn append<'a>( + &'a self, + member: NodeId, + append: AppendRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut message = request(Operation::Append, append.leader_session, append.log_epoch); + message.frames = append + .frames + .into_iter() + .map(|frame| frame.to_vec()) + .collect(); + message.covered_through = append.covered_through; + let reply = self.round_trip(member, message).await?; + Ok(FollowerReceipt { + base_sequence: reply.base_sequence, + durable_through: reply.durable_through, + }) + }) + } + + fn seal<'a>( + &'a self, + member: NodeId, + seal: SealRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let reply = self + .round_trip( + member, + request(Operation::Seal, seal.leader_session, seal.log_epoch), + ) + .await?; + Ok(FollowerReceipt { + base_sequence: reply.base_sequence, + durable_through: reply.durable_through, + }) + }) + } + + fn retire<'a>( + &'a self, + member: NodeId, + retire: RetireRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut message = request(Operation::Retire, retire.leader_session, retire.log_epoch); + message.covered_through = retire.covered_through; + let reply = self.round_trip(member, message).await?; + Ok(FollowerReceipt { + base_sequence: reply.base_sequence, + durable_through: reply.durable_through, + }) + }) + } + + fn tail<'a>(&'a self, member: NodeId, tail: TailRequest) -> BoxFuture<'a, Result>> { + Box::pin(async move { + let page = self.tail_page(member, tail).await?; + if page.next_sequence.is_some() { + return Err(Error::Peer("unbounded follower tail is unsupported")); + } + Ok(page.frames) + }) + } + + fn tail_page<'a>( + &'a self, + member: NodeId, + tail: TailRequest, + ) -> BoxFuture<'a, Result> { + Box::pin(async move { + let mut message = request(Operation::TailPage, tail.leader_session, tail.log_epoch); + message.first_sequence = tail.first_sequence; + let reply = self.round_trip(member, message).await?; + Ok(FollowerTailPage { + frames: reply.frames.into_iter().map(Bytes::from).collect(), + next_sequence: reply.next_sequence, + }) + }) + } +} + +/// Serves one node's durable store on the private Compose network. +pub(super) fn serve( + listener: TcpListener, + member: NodeId, + store: FollowerStore, + directory: NodeDirectory, + key: SigningKey, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let slots = Arc::new(Semaphore::new(512)); + while let Ok((socket, _)) = listener.accept().await { + let Ok(slot) = Arc::clone(&slots).try_acquire_owned() else { + continue; + }; + let store = store.clone(); + let directory = directory.clone(); + let key = key.clone(); + tokio::spawn(async move { + let _slot = slot; + let _ = tokio::time::timeout( + REQUEST_TIMEOUT, + serve_one(socket, member, store, directory, key), + ) + .await; + }); + } + }) +} + +async fn serve_one( + mut socket: TcpStream, + member: NodeId, + store: FollowerStore, + directory: NodeDirectory, + key: SigningKey, +) -> Result<()> { + let encoded = receive(&mut socket, MAX_REQUEST_BYTES).await?; + let envelope = SignedWire::decode(encoded.as_slice()) + .map_err(|_| Error::Peer("invalid follower envelope"))?; + let request = RequestWire::decode(envelope.body.as_slice()) + .map_err(|_| Error::Peer("invalid follower request"))?; + let sender = session(&request.sender)?; + let leader = session(&request.leader)?; + validate_request(&request, member, now_ms())?; + let enrolled = directory + .load(sender, now_ms()) + .await? + .ok_or(Error::PeerAuthorization("follower sender is not live"))?; + verify( + &encoded, + enrolled.advertisement().verifying_key()?, + REQUEST_DOMAIN, + )?; + let digest = blake3::hash(&envelope.body); + let result = execute(member, store, directory, sender, leader, request).await; + let mut reply = ResponseWire { + member: member.as_bytes().to_vec(), + request_digest: digest.as_bytes().to_vec(), + status: 0, + base_sequence: 0, + durable_through: 0, + frames: Vec::new(), + next_sequence: None, + }; + match result { + Ok(Reply::Receipt(receipt)) => { + reply.base_sequence = receipt.base_sequence; + reply.durable_through = receipt.durable_through; + } + Ok(Reply::Page(page)) => { + reply.frames = page + .frames + .into_iter() + .map(|frame| frame.to_vec()) + .collect(); + reply.next_sequence = page.next_sequence; + } + Err(Error::PeerAuthorization(_)) | Err(Error::Fenced) => reply.status = 1, + Err(_) => reply.status = 2, + } + let encoded = signed(reply.encode_to_vec(), &key, RESPONSE_DOMAIN); + send(&mut socket, &encoded, MAX_RESPONSE_BYTES).await +} + +enum Reply { + Receipt(FollowerReceipt), + Page(FollowerTailPage), +} + +async fn execute( + member: NodeId, + store: FollowerStore, + directory: NodeDirectory, + sender: SessionId, + leader: SessionId, + request: RequestWire, +) -> Result { + match Operation::try_from(request.operation)? { + Operation::Append => { + if sender != leader { + return Err(Error::PeerAuthorization( + "invalid follower append sender or batch", + )); + } + directory + .authorize_log_append( + leader, + member, + request.epoch, + request.covered_through, + now_ms(), + ) + .await?; + store + .append( + leader, + request.epoch, + request.frames.into_iter().map(Bytes::from).collect(), + request.covered_through, + ) + .await + .map(Reply::Receipt) + } + Operation::Retire => { + if sender != leader { + return Err(Error::PeerAuthorization( + "invalid follower retirement sender", + )); + } + directory + .authorize_log_retire( + leader, + member, + request.epoch, + request.covered_through, + now_ms(), + ) + .await?; + store + .retire(leader, request.epoch, request.covered_through) + .await + .map(Reply::Receipt) + } + Operation::Seal | Operation::TailPage => { + directory + .authorize_log_recovery(leader, sender, member, request.epoch, now_ms()) + .await?; + if request.operation == Operation::Seal as u32 { + store.seal(leader, request.epoch).await.map(Reply::Receipt) + } else { + store + .read_tail_page(leader, request.epoch, request.first_sequence) + .await + .map(Reply::Page) + } + } + } +} + +/// One serialized authority view shared by heartbeats, enrollment, and log CAS. +#[derive(Clone)] +pub(super) struct ProcessEnrollment { + pub(super) observed: Arc>, + directory: NodeDirectory, + session: SessionId, + observation: Option>, +} + +impl ProcessEnrollment { + pub(super) fn new( + observed: VersionedNodeAdvertisement, + directory: NodeDirectory, + session: SessionId, + observation: Option>, + ) -> Self { + Self { + observed: Arc::new(Mutex::new(observed)), + directory, + session, + observation, + } + } + + fn record_log_event(&self, epoch: u64, phase: &'static str, through: u64) { + if let Some(observation) = &self.observation { + observation.record_node_log_event(epoch, phase, through); + } + } + + pub(super) async fn refresh(&self, next: NodeAdvertisement) -> Result { + let mut observed = self.observed.lock().await; + *observed = self.directory.refresh(&observed, next, now_ms()).await?; + Ok(observed.advertisement().expires_at_ms()) + } + + pub(super) async fn withdraw(&self) -> Result<()> { + let observed = self.observed.lock().await; + self.directory + .withdraw_after_drain(&observed, now_ms()) + .await + } + + pub(super) async fn progress(&self) -> u64 { + self.observed.lock().await.advertisement().progress() + } + + pub(super) async fn generation(&self) -> u64 { + self.observed.lock().await.advertisement().generation() + } +} + +impl NodeLogAuthority for ProcessEnrollment { + fn activate<'a>(&'a self, epoch: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let mut observed = self.observed.lock().await; + if observed + .advertisement() + .log() + .is_none_or(|log| log.epoch() != epoch) + { + return Err(Error::Fenced); + } + *observed = self.directory.activate_log(&observed, now_ms()).await?; + self.record_log_event(epoch, "active", 0); + Ok(()) + }) + } + + fn advance_coverage<'a>(&'a self, epoch: u64, through: u64) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let mut observed = self.observed.lock().await; + if observed + .advertisement() + .log() + .is_none_or(|log| log.epoch() != epoch) + { + return Err(Error::Fenced); + } + *observed = self + .directory + .advance_log_coverage(&observed, through, now_ms()) + .await?; + self.record_log_event(epoch, "coverage", through); + Ok(()) + }) + } + + fn close<'a>(&'a self, barrier: &'a NodeLogRotationBarrier) -> BoxFuture<'a, Result<()>> { + Box::pin(async move { + let mut observed = self.observed.lock().await; + if observed + .advertisement() + .log() + .is_none_or(|log| log.epoch() != barrier.log_epoch()) + { + return Err(Error::Fenced); + } + *observed = self + .directory + .close_log(&observed, barrier, now_ms()) + .await?; + self.record_log_event(barrier.log_epoch(), "closed", barrier.covered_through()); + Ok(()) + }) + } +} + +pub(super) struct ProcessDurabilityProvider { + enrollment: ProcessEnrollment, + key: SigningKey, + lease: NodeLeaseGuard, + telemetry: CellTelemetryHandle, + observation: Arc, +} + +impl ProcessDurabilityProvider { + pub(super) fn new( + enrollment: ProcessEnrollment, + key: SigningKey, + lease: NodeLeaseGuard, + telemetry: CellTelemetryHandle, + observation: Arc, + ) -> Self { + Self { + enrollment, + key, + lease, + telemetry, + observation, + } + } +} + +impl NodeDurabilityProvider for ProcessDurabilityProvider { + fn recruit( + self: Arc, + limits: Limits, + required_follower_bytes: u64, + live_node_limit: usize, + ) -> Pin>> + Send>> { + Box::pin(async move { + let result: Result> = async { + let mut observed = self.enrollment.observed.lock().await; + if observed.advertisement().log().is_none() { + // Keep the measured ensemble shape fixed at both followers. + if self + .enrollment + .directory + .live(now_ms(), live_node_limit) + .await? + .len() + < 3 + { + return Ok(None); + } + let next = self + .enrollment + .directory + .try_recruit_log( + &observed, + 1, + required_follower_bytes, + live_node_limit, + now_ms(), + ) + .await?; + let Some(next) = next else { + return Ok(None); + }; + *observed = next; + self.enrollment.record_log_event(1, "enrolled", 0); + } + let log = observed + .advertisement() + .log() + .ok_or(Error::Node("enrolled follower log disappeared"))?; + let members = log.members().to_vec(); + if members.len() != 2 { + return Err(Error::Node("capacity lane requires two follower members")); + } + let epoch = log.epoch(); + let mut enrolled = HashMap::new(); + for &member in &members { + let advertisement = self + .enrollment + .directory + .resolve_node(member, now_ms()) + .await? + .ok_or(Error::Node("follower member is no longer live"))?; + enrolled.insert(member, advertisement); + } + let transport: Arc = Arc::new(ProcessFollowerTransport::new( + self.enrollment.session, + self.key.clone(), + self.enrollment.directory.clone(), + enrolled, + Some(Arc::clone(&self.observation)), + )?); + let authority: Arc = Arc::new(self.enrollment.clone()); + NodeDurabilityConfig::new( + self.enrollment.session, + NodeId::from_bytes(*self.enrollment.session.as_bytes()), + epoch, + members, + transport, + authority, + self.lease.clone(), + limits, + self.telemetry.clone(), + ) + .map(Some) + } + .await; + result.map_err(|source| Box::new(source) as Box) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use cellule_ltx::{Db, NodeFrameScope, encode_node_frame}; + use cellule_runtime::identity::Digest; + use cellule_runtime::ltx::CellStorageLayout; + use cellule_runtime::node::log_recovery::NodeLogRecovery; + use cellule_runtime::node::{NODE_LOG_PROTOCOL_VERSION, NodeCapacity, NodeFailureDomain}; + use cellule_store::Store; + use object_store::{memory::InMemory, path::Path}; + + fn advertisement( + node: NodeId, + session: SessionId, + key: &SigningKey, + endpoint: String, + issued_at: i64, + lifetime_ms: i64, + ) -> NodeAdvertisement { + NodeAdvertisement::sign( + node, + session, + endpoint, + Digest::from_bytes([10; 32]), + Digest::from_bytes([11; 32]), + Digest::from_bytes([12; 32]), + Digest::from_bytes([13; 32]), + key, + 1, + issued_at, + issued_at + lifetime_ms, + vec![Digest::from_bytes([14; 32])], + vec![1], + NodeFailureDomain::default(), + NodeCapacity { + free_memory_bytes: 1 << 20, + free_disk_bytes: 1 << 20, + follower_free_bytes: 1 << 20, + job_credits: 4, + log_protocol: NODE_LOG_PROTOCOL_VERSION, + ..NodeCapacity::default() + }, + ) + .unwrap() + } + + fn frame(limits: Limits) -> Bytes { + let source = tempfile::TempDir::new().unwrap(); + let mut database = Db::open(&source.path().join("cell.sqlite"), limits).unwrap(); + database + .transaction(|transaction| { + transaction.execute_batch( + "CREATE TABLE events(id INTEGER PRIMARY KEY, body TEXT NOT NULL);\ + INSERT INTO events(body) VALUES ('survives')", + ) + }) + .unwrap(); + let capture = database.capture().unwrap(); + let segment = capture.segments.first().unwrap(); + let encoded = encode_node_frame( + NodeFrameScope { + leader_session: [1; 16], + log_epoch: 1, + node_sequence: 1, + application: [3; 16], + cell: [4; 32], + incarnation: [5; 16], + cell_epoch: 1, + commit_sequence: 1, + }, + segment.info().clone(), + Bytes::from(std::fs::read(segment.path()).unwrap()), + limits, + ) + .unwrap() + .encoded() + .clone(); + database.close().unwrap(); + encoded + } + + #[test] + fn signed_envelope_binds_sender_key_domain_and_body() { + let key = SigningKey::from_bytes(&[7; 32]); + let other = SigningKey::from_bytes(&[8; 32]); + let encoded = signed(vec![1, 2, 3], &key, REQUEST_DOMAIN); + assert_eq!( + verify(&encoded, key.verifying_key(), REQUEST_DOMAIN).unwrap(), + vec![1, 2, 3] + ); + assert!(verify(&encoded, other.verifying_key(), REQUEST_DOMAIN).is_err()); + assert!(verify(&encoded, key.verifying_key(), RESPONSE_DOMAIN).is_err()); + let mut changed = SignedWire::decode(encoded.as_slice()).unwrap(); + changed.body.push(4); + assert!( + verify( + &changed.encode_to_vec(), + key.verifying_key(), + REQUEST_DOMAIN + ) + .is_err() + ); + } + + #[test] + fn request_scope_rejects_wrong_member_stale_epoch_deadline_and_batch() { + let member = NodeId::from_bytes([1; 16]); + let leader = SessionId::from_bytes([2; 16]); + let mut append = request(Operation::Append, leader, 1); + append.member = member.as_bytes().to_vec(); + append.frames.push(vec![1]); + append.deadline_ms = 11_000; + assert_eq!( + validate_request(&append, member, 10_000).unwrap(), + Operation::Append + ); + append.member = NodeId::from_bytes([3; 16]).as_bytes().to_vec(); + assert!(validate_request(&append, member, 10_000).is_err()); + append.member = member.as_bytes().to_vec(); + append.epoch = 0; + assert!(validate_request(&append, member, 10_000).is_err()); + append.epoch = 1; + append.deadline_ms = 10_000; + assert!(validate_request(&append, member, 10_000).is_err()); + append.deadline_ms = 20_001; + assert!(validate_request(&append, member, 10_000).is_err()); + append.deadline_ms = 11_000; + append.frames = vec![vec![1]; MAX_APPEND_FRAMES + 1]; + assert!(validate_request(&append, member, 10_000).is_err()); + append.operation = Operation::Retire as u32; + assert!(validate_request(&append, member, 10_000).is_err()); + } + + #[tokio::test] + async fn oversized_message_is_rejected_before_network_write() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let mut socket = TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + assert!( + send( + &mut socket, + &vec![1; MAX_REQUEST_BYTES + 1], + MAX_REQUEST_BYTES + ) + .await + .is_err() + ); + } + + #[tokio::test] + async fn owner_loss_seals_and_reads_exact_unpublished_network_tail() { + let limits = Limits::default(); + let layout = CellStorageLayout::new( + Store::new(Arc::new(InMemory::new())), + Path::from("follower-network-test"), + [15; 16], + ); + let directory = NodeDirectory::new( + layout, + Digest::from_bytes([10; 32]), + Digest::from_bytes([12; 32]), + Digest::from_bytes([13; 32]), + ); + let leader = SessionId::from_bytes([1; 16]); + let member = NodeId::from_bytes([2; 16]); + let claimant = SessionId::from_bytes([3; 16]); + let leader_key = SigningKey::from_bytes(&[21; 32]); + let member_key = SigningKey::from_bytes(&[22; 32]); + let claimant_key = SigningKey::from_bytes(&[23; 32]); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let follower_port = listener.local_addr().unwrap().port(); + let gateway_port = follower_port.checked_sub(1).unwrap(); + let endpoint = format!("https://127.0.0.1:{gateway_port}"); + let issued_at = now_ms(); + let leader_record = directory + .create( + advertisement( + NodeId::from_bytes([1; 16]), + leader, + &leader_key, + "https://127.0.0.1:8080".into(), + issued_at, + 5_000, + ), + issued_at, + ) + .await + .unwrap(); + let member_record = directory + .create( + advertisement( + member, + SessionId::from_bytes([2; 16]), + &member_key, + endpoint, + now_ms(), + 30_000, + ), + now_ms(), + ) + .await + .unwrap(); + let enrolled = directory + .recruit_log(&leader_record, 1, 1, 3, now_ms()) + .await + .unwrap(); + let root = tempfile::TempDir::new().unwrap(); + let store = FollowerStore::open( + root.path().join("follower"), + limits, + cellule_ltx::DiskBudget::new(1 << 20), + ) + .unwrap(); + let server = serve(listener, member, store, directory.clone(), member_key); + let members = HashMap::from([(member, member_record.advertisement().clone())]); + let transport = ProcessFollowerTransport::new( + leader, + leader_key.clone(), + directory.clone(), + members.clone(), + None, + ) + .unwrap(); + assert!( + transport + .append( + NodeId::from_bytes([9; 16]), + AppendRequest { + leader_session: leader, + log_epoch: 1, + frames: vec![Bytes::from_static(b"untrusted")], + covered_through: 0, + }, + ) + .await + .is_err() + ); + let expected = frame(limits); + let receipt = transport + .append( + member, + AppendRequest { + leader_session: leader, + log_epoch: 1, + frames: vec![expected.clone()], + covered_through: 0, + }, + ) + .await + .unwrap(); + assert_eq!(receipt.durable_through, 1); + let mut interrupted = request(Operation::Append, leader, 1); + interrupted.sender = leader.as_bytes().to_vec(); + interrupted.member = member.as_bytes().to_vec(); + interrupted.frames = vec![expected.to_vec()]; + interrupted.deadline_ms = now_ms() + MAX_DEADLINE_AHEAD_MS; + let mut socket = TcpStream::connect(("127.0.0.1", follower_port)) + .await + .unwrap(); + send( + &mut socket, + &signed(interrupted.encode_to_vec(), &leader_key, REQUEST_DOMAIN), + MAX_REQUEST_BYTES, + ) + .await + .unwrap(); + drop(socket); + let retried = transport + .append( + member, + AppendRequest { + leader_session: leader, + log_epoch: 1, + frames: vec![expected.clone()], + covered_through: 0, + }, + ) + .await + .unwrap(); + assert_eq!(retried.durable_through, 1); + let enrollment = ProcessEnrollment::new(enrolled, directory.clone(), leader, None); + let next = advertisement( + NodeId::from_bytes([1; 16]), + leader, + &leader_key, + "https://127.0.0.1:8080".into(), + now_ms(), + 5_000, + ); + let (activated, refreshed) = tokio::join!(enrollment.activate(1), enrollment.refresh(next)); + activated.unwrap(); + refreshed.unwrap(); + assert!( + directory + .load(leader, now_ms()) + .await + .unwrap() + .unwrap() + .advertisement() + .log() + .unwrap() + .active() + ); + assert!( + transport + .append( + member, + AppendRequest { + leader_session: leader, + log_epoch: 2, + frames: vec![expected.clone()], + covered_through: 0, + } + ) + .await + .is_err() + ); + assert!( + transport + .retire( + member, + RetireRequest { + leader_session: leader, + log_epoch: 1, + covered_through: 1, + } + ) + .await + .is_err() + ); + let claimant_record = directory + .create( + advertisement( + NodeId::from_bytes([3; 16]), + claimant, + &claimant_key, + "https://127.0.0.1:8082".into(), + now_ms(), + 30_000, + ), + now_ms(), + ) + .await + .unwrap(); + let _ = claimant_record; + tokio::time::sleep(Duration::from_millis(5_100)).await; + let fenced = directory + .claim_expired(leader, claimant, now_ms()) + .await + .unwrap(); + assert_eq!(fenced.log().unwrap().tiered_through(), 0); + let recovery_transport: Arc = Arc::new( + ProcessFollowerTransport::new(claimant, claimant_key, directory, members, None) + .unwrap(), + ); + let recovery = NodeLogRecovery::from_fenced(recovery_transport, &fenced, limits).unwrap(); + let sealed = recovery.ensure_sealed().await.unwrap(); + assert_eq!(sealed.frame_count(), 1); + assert_eq!(sealed.frames[0].encoded(), &expected); + server.abort(); + let _ = server.await; + } +} diff --git a/crates/cellule-app/tests/process_node.rs b/crates/cellule-app/tests/process_node.rs index 0ef3b9c..ee2906b 100644 --- a/crates/cellule-app/tests/process_node.rs +++ b/crates/cellule-app/tests/process_node.rs @@ -1,8 +1,9 @@ //! Public host and authoritative session lifecycle for the process fixture. use super::performance_fixture::{DurabilityRecorder, node_session, now_ms}; +use super::process_follower::{ProcessDurabilityProvider, ProcessEnrollment}; use crate::*; -use cellule_host::{CellNode, CellNodeBuilder}; +use cellule_host::{CellNode, CellNodeBuilder, NodeDurabilitySupervisorConfig}; use cellule_runtime::node::lease::NodeLeaseGuard; use std::time::Duration; use tokio_util::sync::CancellationToken; @@ -16,6 +17,12 @@ pub(super) fn directory(layout: &CellStorageLayout, registry: &Registry) -> Node ) } +pub(super) fn signing_key(node: usize) -> SigningKey { + let mut seed = [93; 32]; + seed[0] = u8::try_from(node).unwrap(); + SigningKey::from_bytes(&seed) +} + pub(super) async fn start( node: usize, application: Arc, @@ -26,6 +33,21 @@ pub(super) async fn start( CellNode, Arc, cellule_host::read_replicas::ReadReplicaManager, +) { + start_configured(node, application, layout, root, endpoint, false).await +} + +pub(super) async fn start_configured( + node: usize, + application: Arc, + layout: &CellStorageLayout, + root: &std::path::Path, + endpoint: String, + follower_enabled: bool, +) -> ( + CellNode, + Arc, + cellule_host::read_replicas::ReadReplicaManager, ) { let registry = application.registry(); // Keep writer admission at 32 Cells while charging both the old and new @@ -34,16 +56,23 @@ pub(super) async fn start( .unwrap() .with_native_memory_limit(32 << 20) .unwrap(); - let host = CellNodeBuilder::new(application) + let mut builder = CellNodeBuilder::new(application) .with_runtime(pool, 64 * 1024 * 1024) .with_session(node_session(node)) - .with_replica_host(reference_host()) - .build() - .unwrap(); + .with_replica_host(reference_host()); + if follower_enabled { + builder = builder.with_follower_store( + root.join("followers"), + Limits::default(), + DiskBudget::new(1 << 30), + ); + } + let host = builder.build().unwrap(); let durability = Arc::new(DurabilityRecorder::default()); host.install_telemetry(durability.clone()).unwrap(); let directory = directory(layout, ®istry); - let signer = SigningKey::from_bytes(&[93; 32]); + let signer = signing_key(node); + let follower_signer = signer.clone(); let advertisement = move |now: i64, progress| { NodeAdvertisement::sign( NodeId::from_bytes(*node_session(node).as_bytes()), @@ -64,22 +93,56 @@ pub(super) async fn start( free_memory_bytes: 64 * 1024 * 1024, free_disk_bytes: 1 << 30, job_credits: 32, + follower_free_bytes: if follower_enabled { 1 << 30 } else { 0 }, + log_protocol: if follower_enabled { + cellule_runtime::node::NODE_LOG_PROTOCOL_VERSION + } else { + 0 + }, ..NodeCapacity::default() }, ) }; let now = now_ms(); - let mut observed = directory + let observed = directory .create(advertisement(now, 1).unwrap(), now) .await .unwrap(); let lease = NodeLeaseGuard::new(now_ms(), observed.advertisement().expires_at_ms()).unwrap(); + let enrollment = ProcessEnrollment::new( + observed, + directory.clone(), + node_session(node), + follower_enabled.then(|| Arc::clone(&durability)), + ); let shutdown = CancellationToken::new(); let tasks = host .install_task_group(CancellationToken::new(), shutdown.clone()) .unwrap(); host.install_node_lease_for_startup(lease.clone()).unwrap(); + if follower_enabled { + let provider = Arc::new(ProcessDurabilityProvider::new( + enrollment.clone(), + follower_signer, + lease.clone(), + host.runtime().telemetry_handle(), + Arc::clone(&durability), + )); + let configuration = NodeDurabilitySupervisorConfig::new( + ApplicationId::from_bytes([82; 16]), + Limits::default(), + 1 << 20, + 32, + Duration::from_millis(250), + Duration::from_secs(1), + 1_000_000, + ) + .unwrap(); + host.install_node_durability_provider(provider, configuration) + .unwrap(); + } let (renewed, first_renewal) = tokio::sync::oneshot::channel(); + let renewing = enrollment.clone(); tasks .spawn_lease_maintenance(async move { let mut renewed = Some(renewed); @@ -90,29 +153,24 @@ pub(super) async fn start( () = tokio::time::sleep(Duration::from_secs(5)) => {} } let now = now_ms(); - let next = advertisement(now, observed.advertisement().progress() + 1)?; + let next = advertisement(now, renewing.progress().await + 1)?; // Finish the conditional write before observing shutdown so // withdrawal always uses the latest acknowledged generation. - observed = tokio::time::timeout( - lease.remaining(), - directory.refresh(&observed, next, now), - ) - .await - .map_err(|_| Error::Deadline)??; - lease.renew(now_ms(), observed.advertisement().expires_at_ms())?; + let expires_at = + tokio::time::timeout(lease.remaining(), renewing.refresh(next)) + .await + .map_err(|_| Error::Deadline)??; + lease.renew(now_ms(), expires_at)?; if let Some(renewed) = renewed.take() { let _ = renewed.send(()); } } - tokio::time::timeout( - Duration::from_secs(20), - directory.withdraw(&observed, now_ms()), - ) - .await - .map_err(|_| Error::Deadline)??; + tokio::time::timeout(Duration::from_secs(20), renewing.withdraw()) + .await + .map_err(|_| Error::Deadline)??; println!( "PERF node_{node}_session_withdrawn: generation={}", - observed.advertisement().generation() + renewing.generation().await ); Ok(()) } @@ -142,7 +200,7 @@ pub(super) async fn start( Arc::new(cellule_runtime::peer::PeerSigner::new( node_session(node), host.application().registry().release_digest(), - SigningKey::from_bytes(&[93; 32]), + signing_key(node), )), cellule_runtime::peer::PeerPrincipal { issuer: "reference-runtime".into(), diff --git a/crates/cellule-ltx/tests/cell/roots/directory.rs b/crates/cellule-ltx/tests/cell/roots/directory.rs index d398ba0..5d9c551 100644 --- a/crates/cellule-ltx/tests/cell/roots/directory.rs +++ b/crates/cellule-ltx/tests/cell/roots/directory.rs @@ -106,6 +106,9 @@ async fn directory_cache_survives_replica_restart_without_directory_origin_read( let incarnation = [86; 16]; let cache_host = Host::default() .with_local_disk_budget(DiskBudget::new(64 * 1024 * 1024)) + // Optional cache fills use try-acquire. Give this fixture its own + // admission slots so parallel tests cannot suppress the warm fill. + .with_job_slots(Arc::new(tokio::sync::Semaphore::new(4))) .with_directory_cache(cache_root.clone()) .await .unwrap(); @@ -146,6 +149,7 @@ async fn directory_cache_survives_replica_restart_without_directory_origin_read( })); let cached_host = Host::default() .with_local_disk_budget(DiskBudget::new(64 * 1024 * 1024)) + .with_job_slots(Arc::new(tokio::sync::Semaphore::new(4))) .with_directory_cache(cache_root) .await .unwrap(); diff --git a/crates/cellule-peer-http/Cargo.toml b/crates/cellule-peer-http/Cargo.toml index 5d18ba4..59acb36 100644 --- a/crates/cellule-peer-http/Cargo.toml +++ b/crates/cellule-peer-http/Cargo.toml @@ -25,6 +25,6 @@ url = "2" x509-cert = "0.2.5" [dev-dependencies] -cellule-store.workspace = true +cellule-store = { workspace = true, features = ["test-support"] } object_store.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/cellule-peer-http/docs/routing.md b/crates/cellule-peer-http/docs/routing.md index e0bdd35..934b6e4 100644 --- a/crates/cellule-peer-http/docs/routing.md +++ b/crates/cellule-peer-http/docs/routing.md @@ -4,6 +4,14 @@ envelope, and bounds request and response bytes. Both owner-routed and direct-node calls reject oversized requests and zero deadlines before lookup. +The sender shares a private, 4,096-entry owner hint across its clones. A hint +is populated only after exact control and signed node-record validation. It +expires after at most five seconds, and at least one second before the signed +node lease. A proven not-started refusal drops the attempted session and +forces one exact refresh within the original deadline. An invalid or lost +response remains an unknown outcome and is never resent blindly. The receiver +still authorizes the peer and fences stale owners. + | Response | Meaning | | --- | --- | | Valid reply | Return the exact peer result. | @@ -27,3 +35,50 @@ sequenceDiagram The same timeout governs resolution, sending, and pacing. A delay that consumes the budget returns a deadline error without sleeping. Other invalid results are never guessed to have failed before execution. + +## Local adapter comparison, 2026-09-29 + +Run the ignored `tests::owner_lookup_performance` benchmark exactly once per +invocation after checking its selector with `-- --list`: + +```sh +CARGO_TARGET_DIR="$HOME/Workspace/crabbuild-target/cellule-route-5ca5" \ + cargo test -p cellule-peer-http tests::owner_lookup_performance --locked \ + -- --ignored --exact --nocapture +``` + +The baseline used `dc387a8` in an isolated source snapshot with only the +benchmark fixture and its test dependency added. The candidate used the same +revision plus the worktree's owner-hint change. Both used a validated signed +owner advertisement, a counting in-memory object store, and local mTLS Axum +peer. Each lane has 1,024 requests at concurrency 1 and 16. The full-adapter +lane calls `send_inner`; the lookup and HTTP lanes isolate its components. +The server returns an encoded peer reply without running receiver dispatch or +checking the request envelope. The raw TSVs and binary digests are retained under +`$HOME/Workspace/crabbuild-target/cellule-route-5ca5/evidence`. + +| Pair | Full adapter p95, c=1 baseline → candidate | p99, c=1 | p95, c=16 | p99, c=16 | +| --- | ---: | ---: | ---: | ---: | +| 1 | 5.874 → 1.887 ms | 8.378 → 3.714 ms | 38.966 → 9.543 ms | 68.701 → 11.669 ms | +| 2 | 9.395 → 2.215 ms | 18.612 → 4.689 ms | 56.168 → 21.710 ms | 82.897 → 35.607 ms | +| 3 | 12.864 → 4.798 ms | 34.210 → 7.957 ms | 92.130 → 25.913 ms | 128.453 → 38.042 ms | + +All three final pairs cleared the 10% p95 gate at both concurrency levels, +improved p99, and raised fully successful concurrency-16 adapter throughput +from 245–409 to 1,106–2,547 requests/s. Warm full-adapter sender reads fell +from two body reads per request to zero while the hint remained live. One of +six candidate warm lanes had two reads across 1,024 calls because its +five-second hint expired during the measurement. Cold sends still made two +metadata reads in both builds. Earlier exploratory runs, including one with +a candidate p99 network spike, remain in the evidence directory; this shared +workstation is not a production latency profile. Local HTTP/TLS is now the +dominant measured adapter phase. This fixture does not measure product ingress, +RustFS, a remote network, or owner SQL/publication time. + +The embedding application maintainer should check whether its ingress route +shares the sender hint and passes a known description through +`with_observed_description`. A product comparison should schedule the same +local and forwarded actions across hot and many-Cell stages, retain owner-loss +and receipt evidence, and count object-store requests per action. Write +throughput attribution is tracked separately by the entity workload and the +[write-capacity report](../../cellule-app/performance/2026-09-29-write-capacity.md). diff --git a/crates/cellule-peer-http/src/lib.rs b/crates/cellule-peer-http/src/lib.rs index a94bfbc..d7a8358 100644 --- a/crates/cellule-peer-http/src/lib.rs +++ b/crates/cellule-peer-http/src/lib.rs @@ -5,7 +5,7 @@ mod tls; pub use tls::{LoadedPeerTls, PeerTlsClient, PeerTlsIdentity, PeerTlsListener, TlsError}; use std::{ - collections::VecDeque, + collections::{HashMap, VecDeque}, future::Future, pin::Pin, sync::{Arc, Mutex}, @@ -15,7 +15,7 @@ use std::{ use cellule_runtime::Error as CellError; use cellule_runtime::cell::application::ApplicationIdentity; use cellule_runtime::control::authority::CellAuthority; -use cellule_runtime::identity::{CellTarget, Digest, SessionId}; +use cellule_runtime::identity::{CellId, CellTarget, Digest, SessionId}; use cellule_runtime::node::{NodeAdvertisement, NodeDirectory}; use cellule_runtime::peer::{PeerRoundTrip, wire as peer_wire}; use futures_util::StreamExt; @@ -23,6 +23,9 @@ use http::{StatusCode, header}; const PEER_FORWARD_PATH: &str = "internal/cells/v1/forward"; const MAX_PEER_CLIENTS: usize = 1_024; +const MAX_OWNER_HINTS: usize = 4_096; +const OWNER_HINT_LIFETIME: Duration = Duration::from_secs(5); +const OWNER_LEASE_MARGIN_MS: i64 = 1_000; /// Content type accepted by the private peer forwarding endpoint. pub const PROTOBUF_MEDIA_TYPE: &str = "application/x-protobuf"; @@ -61,6 +64,7 @@ pub struct PeerHttpRoundTrip { tls: Arc, session: SessionId, clients: Arc>>, + owners: Arc>>, } impl PeerHttpRoundTrip { @@ -80,6 +84,7 @@ impl PeerHttpRoundTrip { tls, session, clients: Arc::new(Mutex::new(VecDeque::new())), + owners: Arc::new(Mutex::new(HashMap::new())), } } @@ -106,23 +111,34 @@ impl PeerHttpRoundTrip { } } let remaining_ms = remaining_timeout(started, timeout_ms)?; - let owner = tokio::time::timeout( - Duration::from_millis(u64::from(remaining_ms)), - self.owner(&target), - ) - .await - .map_err(|_| CellError::Deadline)??; + let owner = + tokio::time::timeout(Duration::from_millis(u64::from(remaining_ms)), async { + if last_retry.is_some() { + self.refresh_owner(&target).await + } else { + self.owner(&target).await + } + }) + .await + .map_err(|_| CellError::Deadline)??; let remaining_ms = remaining_timeout(started, timeout_ms)?; match self.send_once(&owner, request.clone(), remaining_ms).await { Ok(PeerHttpAttempt::Reply(reply)) => return Ok(reply), - Ok(PeerHttpAttempt::Retry(error, delay)) => last_retry = Some((error, delay)), + Ok(PeerHttpAttempt::Retry(error, delay)) => { + self.invalidate_owner(target.cell_id(), owner.session); + last_retry = Some((error, delay)); + } Ok(PeerHttpAttempt::Unknown(error)) => { + self.invalidate_owner(target.cell_id(), owner.session); return Err(CellError::PeerTransportUnknown { context: "peer HTTP response was lost or invalid", source: Box::new(error), }); } - Err(error) => return Err(error), + Err(error) => { + self.invalidate_owner(target.cell_id(), owner.session); + return Err(error); + } } } Err(last_retry.map_or(CellError::CellNotActive, |(error, _)| error)) @@ -162,6 +178,20 @@ impl PeerHttpRoundTrip { async fn owner(&self, target: &CellTarget) -> cellule_runtime::Result { self.scope.check_target(target)?; + let now = now_ms()?; + if let Some(owner) = self.cached_owner(target.cell_id(), now) { + return Ok(owner); + } + self.load_owner(target).await + } + + async fn refresh_owner(&self, target: &CellTarget) -> cellule_runtime::Result { + self.scope.check_target(target)?; + self.load_owner(target).await + } + + async fn load_owner(&self, target: &CellTarget) -> cellule_runtime::Result { + let lookup_started = Instant::now(); let control = self .authority .load(target.cell_id()) @@ -187,12 +217,116 @@ impl PeerHttpRoundTrip { "Cell owner endpoint is not enrolled", )); } - Ok(RemotePeer { + let remote = RemotePeer { session: owner.session, endpoint: url::Url::parse(advertisement.endpoint()).map_err(peer_transport)?, certificate: advertisement.certificate(), public_key: advertisement.verifying_key()?.to_bytes(), - }) + }; + self.remember_owner( + target.cell_id(), + remote.clone(), + advertisement.expires_at_ms(), + now_ms, + lookup_started, + ); + Ok(remote) + } + + fn cached_owner(&self, cell: CellId, now_ms: i64) -> Option { + let Ok(mut owners) = self.owners.lock() else { + return None; + }; + let observed = owners.get(&cell)?; + if observed.expires_at <= Instant::now() + || observed.lease_expires_at_ms <= now_ms.saturating_add(OWNER_LEASE_MARGIN_MS) + { + owners.remove(&cell); + return None; + } + observed.owner.clone() + } + + fn remember_owner( + &self, + cell: CellId, + owner: RemotePeer, + lease_expires_at_ms: i64, + observed_at_ms: i64, + lookup_started: Instant, + ) { + let lease_margin = lease_expires_at_ms + .saturating_sub(observed_at_ms) + .saturating_sub(OWNER_LEASE_MARGIN_MS); + let Ok(lease_margin) = u64::try_from(lease_margin) else { + return; + }; + if lease_margin == 0 { + return; + } + let lifetime = OWNER_HINT_LIFETIME.min(Duration::from_millis(lease_margin)); + let Some(expires_at) = Instant::now().checked_add(lifetime) else { + return; + }; + let Ok(mut owners) = self.owners.lock() else { + return; + }; + if owners + .get(&cell) + .is_some_and(|current| current.lookup_started > lookup_started) + { + return; + } + if owners.len() >= MAX_OWNER_HINTS + && !owners.contains_key(&cell) + && let Some(evicted) = owners.keys().next().copied() + { + owners.remove(&evicted); + } + owners.insert( + cell, + CachedOwner { + owner: Some(owner), + expires_at, + lease_expires_at_ms, + lookup_started, + }, + ); + } + + fn invalidate_owner(&self, cell: CellId, session: SessionId) { + let Ok(mut owners) = self.owners.lock() else { + return; + }; + if owners.get(&cell).is_some_and(|cached| { + cached + .owner + .as_ref() + .is_some_and(|owner| owner.session != session) + }) { + return; + } + let now = Instant::now(); + let Some(expires_at) = now.checked_add(OWNER_HINT_LIFETIME) else { + return; + }; + if owners.len() >= MAX_OWNER_HINTS + && !owners.contains_key(&cell) + && let Some(evicted) = owners.keys().next().copied() + { + owners.remove(&evicted); + } + // Keep a short tombstone so a lookup started before this refusal + // cannot repopulate the invalidated session after the lock is released. + owners.insert( + cell, + CachedOwner { + owner: None, + expires_at, + lease_expires_at_ms: i64::MAX, + lookup_started: now, + }, + ); } fn client(&self, owner: &RemotePeer) -> cellule_runtime::Result { @@ -377,10 +511,12 @@ impl Clone for PeerHttpRoundTrip { tls: Arc::clone(&self.tls), session: self.session, clients: Arc::clone(&self.clients), + owners: Arc::clone(&self.owners), } } } +#[derive(Clone)] struct RemotePeer { session: SessionId, endpoint: url::Url, @@ -388,6 +524,13 @@ struct RemotePeer { public_key: [u8; 32], } +struct CachedOwner { + owner: Option, + expires_at: Instant, + lease_expires_at_ms: i64, + lookup_started: Instant, +} + struct CachedPeerClient { session: SessionId, certificate: Digest, diff --git a/crates/cellule-peer-http/src/tests.rs b/crates/cellule-peer-http/src/tests.rs index d9826a7..efa4722 100644 --- a/crates/cellule-peer-http/src/tests.rs +++ b/crates/cellule-peer-http/src/tests.rs @@ -1,9 +1,13 @@ use super::*; use axum::{Router, body::Body, response::Response, routing::post}; +use cellule_runtime::cell::catalog::{CatalogEntry, CatalogRole, CellCatalog}; +use cellule_runtime::control::{ControlState, Owner, Transition}; +use cellule_runtime::identity::IncarnationId; use cellule_runtime::identity::{ApplicationId, NamespaceId, NodeId, TenantId}; use cellule_runtime::ltx::CellStorageLayout; use cellule_runtime::node::{NodeCapacity, NodeFailureDomain}; use cellule_store::Store; +use cellule_store::test_support::CountingObjectStore; use ed25519_dalek::SigningKey; use object_store::{memory::InMemory, path::Path}; @@ -101,7 +105,10 @@ async fn both_routes_reject_exhausted_deadlines_before_dispatch() { } } -async fn http_attempt(status: StatusCode, delay: Option<&'static str>) -> PeerHttpAttempt { +async fn http_attempt( + status: StatusCode, + delay: Option<&'static str>, +) -> cellule_runtime::Result { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let router = Router::new().route( @@ -133,7 +140,7 @@ async fn http_attempt(status: StatusCode, delay: Option<&'static str>) -> PeerHt let result = transport.send_once(&peer, vec![1], 5_000).await; stop.send(()).unwrap(); server.await.unwrap(); - result.unwrap() + result } #[tokio::test] @@ -143,7 +150,7 @@ async fn admission_responses_preserve_retry_delay() { StatusCode::SERVICE_UNAVAILABLE, ] { assert!(matches!( - http_attempt(status, Some("2")).await, + http_attempt(status, Some("2")).await.unwrap(), PeerHttpAttempt::Retry(CellError::Capacity(_), delay) if delay == Duration::from_secs(2) )); } @@ -153,8 +160,750 @@ async fn admission_responses_preserve_retry_delay() { async fn server_failure_or_invalid_success_remains_unknown() { for status in [StatusCode::INTERNAL_SERVER_ERROR, StatusCode::OK] { assert!(matches!( - http_attempt(status, None).await, + http_attempt(status, None).await.unwrap(), PeerHttpAttempt::Unknown(_) )); } } + +#[tokio::test] +async fn authentication_refusal_is_not_an_owner_retry() { + for status in [StatusCode::UNAUTHORIZED, StatusCode::FORBIDDEN] { + assert!(matches!( + http_attempt(status, None).await, + Err(CellError::PeerAuthorization(_)) + )); + } +} + +async fn owner_lookup_fixture() -> (PeerHttpRoundTrip, CellTarget, Arc) { + owner_lookup_fixture_with_endpoint( + "https://owner.example:443".into(), + Digest::from_bytes([23; 32]), + Digest::from_bytes([23; 32]), + SigningKey::from_bytes(&[27; 32]), + Arc::new(TestClients), + ) + .await +} + +async fn owner_lookup_fixture_with_endpoint( + endpoint: String, + certificate: Digest, + fleet: Digest, + signing_key: SigningKey, + clients: Arc, +) -> (PeerHttpRoundTrip, CellTarget, Arc) { + let application = ApplicationId::from_bytes([21; 16]); + let tenant = TenantId::from_bytes([22; 16]); + let digest = Digest::from_bytes([23; 32]); + let counted = Arc::new(CountingObjectStore::new(Arc::new(InMemory::new()))); + let layout = CellStorageLayout::new( + Store::new(counted.clone()), + Path::from("owner-lookup-performance"), + *application.as_bytes(), + ); + let target = CellTarget::new( + tenant, + application, + NamespaceId::from_bytes([24; 16]), + b"partition", + ) + .unwrap(); + let catalog = CellCatalog::new(layout.clone(), tenant); + let proof = catalog + .provision(CatalogEntry::new(&target, CatalogRole::Sql, digest, 1).unwrap()) + .await + .unwrap(); + let authority = CellAuthority::new(layout.clone()); + let directory = NodeDirectory::new(layout, fleet, digest, digest); + let now = now_ms().unwrap(); + let advertisement = NodeAdvertisement::sign( + NodeId::from_bytes([25; 16]), + SessionId::from_bytes([26; 16]), + endpoint, + fleet, + certificate, + digest, + digest, + &signing_key, + 1, + now, + now + 30_000, + vec![digest], + vec![1], + NodeFailureDomain::default(), + NodeCapacity::default(), + ) + .unwrap(); + directory.create(advertisement.clone(), now).await.unwrap(); + authority + .create_initial( + &proof, + IncarnationId::from_bytes([28; 16]), + Owner { + session: advertisement.session(), + endpoint: advertisement.endpoint().to_owned(), + }, + ) + .await + .unwrap(); + counted.reset(); + let transport = PeerHttpRoundTrip::new( + Arc::new(ApplicationIdentity::new(tenant, application)), + authority, + directory, + clients, + SessionId::from_bytes([29; 16]), + ); + (transport, target, counted) +} + +#[tokio::test] +async fn exact_owner_lookup_reads_control_and_signed_session() { + let (transport, target, counted) = owner_lookup_fixture().await; + let owner = transport.owner(&target).await.unwrap(); + assert_eq!(owner.session, SessionId::from_bytes([26; 16])); + assert_eq!(counted.counts().body_requests(), 2); + transport.owner(&target).await.unwrap(); + assert_eq!(counted.counts().body_requests(), 2); +} + +#[tokio::test] +async fn invalidated_hint_resolves_a_signed_owner_after_authority_takeover() { + let (transport, target, counted) = owner_lookup_fixture().await; + let old = transport.owner(&target).await.unwrap(); + let now = now_ms().unwrap(); + let successor = NodeAdvertisement::sign( + NodeId::from_bytes([35; 16]), + SessionId::from_bytes([36; 16]), + "https://successor.example:443".into(), + Digest::from_bytes([23; 32]), + Digest::from_bytes([23; 32]), + Digest::from_bytes([23; 32]), + Digest::from_bytes([23; 32]), + &SigningKey::from_bytes(&[37; 32]), + 1, + now, + now + 30_000, + vec![Digest::from_bytes([23; 32])], + vec![1], + NodeFailureDomain::default(), + NodeCapacity::default(), + ) + .unwrap(); + transport + .directory + .create(successor.clone(), now) + .await + .unwrap(); + let observed = transport + .authority + .load(target.cell_id()) + .await + .unwrap() + .unwrap(); + let next = observed + .value() + .takeover(Owner { + session: successor.session(), + endpoint: successor.endpoint().to_owned(), + }) + .unwrap(); + transport + .authority + .transition(&observed, next, Transition::Takeover) + .await + .unwrap(); + counted.reset(); + assert_eq!(transport.owner(&target).await.unwrap().session, old.session); + assert_eq!(counted.counts().body_requests(), 0); + transport.invalidate_owner(target.cell_id(), old.session); + assert_eq!( + transport.owner(&target).await.unwrap().session, + successor.session() + ); + assert_eq!(counted.counts().body_requests(), 2); +} + +#[tokio::test] +async fn retired_owner_session_fails_closed_after_hint_invalidation() { + let (transport, target, counted) = owner_lookup_fixture().await; + let owner = transport.owner(&target).await.unwrap(); + let now = now_ms().unwrap(); + let enrolled = transport + .directory + .load(owner.session, now) + .await + .unwrap() + .unwrap(); + transport.directory.withdraw(&enrolled, now).await.unwrap(); + transport.invalidate_owner(target.cell_id(), owner.session); + counted.reset(); + assert!(transport.owner(&target).await.is_err()); + assert_eq!(counted.counts().body_requests(), 2); +} + +#[tokio::test] +async fn tombstoned_cell_is_not_routed_from_an_invalidated_hint() { + let (transport, target, counted) = owner_lookup_fixture().await; + let old = transport.owner(&target).await.unwrap(); + let observed = transport + .authority + .load(target.cell_id()) + .await + .unwrap() + .unwrap(); + let mut next = observed.value().clone(); + next.epoch += 1; + next.revision += 1; + next.progress += 1; + next.state = ControlState::Tombstoned; + next.owner = None; + transport + .authority + .transition(&observed, next, Transition::Tombstone) + .await + .unwrap(); + transport.invalidate_owner(target.cell_id(), old.session); + counted.reset(); + assert!(matches!( + transport.owner(&target).await, + Err(CellError::CellNotActive) + )); + assert_eq!(counted.counts().body_requests(), 1); +} + +#[tokio::test] +async fn owner_hint_is_shared_scoped_and_invalidated_by_session() { + let (transport, target, counted) = owner_lookup_fixture().await; + let owner = transport.owner(&target).await.unwrap(); + let clone = transport.clone(); + clone.owner(&target).await.unwrap(); + assert_eq!(counted.counts().body_requests(), 2); + + let foreign = CellTarget::new( + TenantId::from_bytes([31; 16]), + target.application(), + target.namespace(), + target.partition(), + ) + .unwrap(); + assert!(matches!( + clone.owner(&foreign).await, + Err(CellError::PeerAuthorization(_)) + )); + assert_eq!(counted.counts().body_requests(), 2); + + transport.invalidate_owner(target.cell_id(), SessionId::from_bytes([32; 16])); + clone.owner(&target).await.unwrap(); + assert_eq!(counted.counts().body_requests(), 2); + transport.invalidate_owner(target.cell_id(), owner.session); + clone.owner(&target).await.unwrap(); + assert_eq!(counted.counts().body_requests(), 4); +} + +#[tokio::test] +async fn delayed_old_lookup_cannot_replace_new_owner_hint() { + let (transport, target, _) = owner_lookup_fixture().await; + let old_started = Instant::now(); + let new_started = old_started + Duration::from_millis(1); + let now = now_ms().unwrap(); + let peer = RemotePeer { + session: SessionId::from_bytes([33; 16]), + endpoint: "https://new.example:443".parse().unwrap(), + certificate: Digest::from_bytes([34; 32]), + public_key: [35; 32], + }; + transport.remember_owner( + target.cell_id(), + peer.clone(), + now + 10_000, + now, + new_started, + ); + let old = RemotePeer { + session: SessionId::from_bytes([36; 16]), + ..peer + }; + transport.remember_owner(target.cell_id(), old, now + 10_000, now, old_started); + assert_eq!( + transport.owner(&target).await.unwrap().session, + peer.session + ); + transport.invalidate_owner(target.cell_id(), peer.session); + transport.remember_owner( + target.cell_id(), + RemotePeer { + session: SessionId::from_bytes([36; 16]), + endpoint: "https://old.example:443".parse().unwrap(), + certificate: Digest::from_bytes([34; 32]), + public_key: [35; 32], + }, + now + 10_000, + now, + old_started, + ); + assert!( + transport + .owners + .lock() + .unwrap() + .get(&target.cell_id()) + .unwrap() + .owner + .is_none() + ); +} + +#[tokio::test] +async fn near_expired_owner_lease_is_not_cached() { + let (transport, target, counted) = owner_lookup_fixture().await; + let now = now_ms().unwrap(); + let peer = RemotePeer { + session: SessionId::from_bytes([37; 16]), + endpoint: "https://old.example:443".parse().unwrap(), + certificate: Digest::from_bytes([38; 32]), + public_key: [39; 32], + }; + transport.remember_owner(target.cell_id(), peer, now + 500, now, Instant::now()); + assert_eq!( + transport.owner(&target).await.unwrap().session, + SessionId::from_bytes([26; 16]) + ); + assert_eq!(counted.counts().body_requests(), 2); +} + +#[tokio::test] +async fn expired_owner_hint_forces_exact_lookup_for_concurrent_callers() { + let (transport, target, counted) = owner_lookup_fixture().await; + transport.owner(&target).await.unwrap(); + transport + .owners + .lock() + .unwrap() + .get_mut(&target.cell_id()) + .unwrap() + .expires_at = Instant::now() - Duration::from_millis(1); + counted.reset(); + let calls = (0..5).map(|_| { + let transport = transport.clone(); + let target = target.clone(); + async move { transport.owner(&target).await.unwrap() } + }); + let owners = futures_util::future::join_all(calls).await; + assert!( + owners + .iter() + .all(|owner| owner.session == owners[0].session) + ); + assert!(counted.counts().body_requests() >= 2); + assert!(counted.counts().body_requests() <= 10); +} + +#[tokio::test] +async fn owner_hint_cache_stays_bounded() { + let (transport, target, _) = owner_lookup_fixture().await; + let owner = transport.owner(&target).await.unwrap(); + let now = now_ms().unwrap(); + for index in 0..MAX_OWNER_HINTS + 32 { + let mut bytes = [0; 32]; + bytes[..8].copy_from_slice(&(index as u64).to_be_bytes()); + transport.remember_owner( + CellId::from_bytes(bytes), + owner.clone(), + now + 10_000, + now, + Instant::now(), + ); + } + assert_eq!(transport.owners.lock().unwrap().len(), MAX_OWNER_HINTS); +} + +#[tokio::test] +async fn ambiguous_peer_response_does_not_retry_cached_route() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let hits = Arc::new(AtomicUsize::new(0)); + let server_hits = Arc::clone(&hits); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let router = Router::new().route( + "/internal/cells/v1/forward", + post(move || { + server_hits.fetch_add(1, Ordering::Relaxed); + async { + Response::builder() + .status(StatusCode::OK) + .body(Body::from("bad")) + .unwrap() + } + }), + ); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + let (transport, target, counted) = owner_lookup_fixture().await; + let now = now_ms().unwrap(); + let peer = RemotePeer { + session: SessionId::from_bytes([26; 16]), + endpoint: format!("http://{address}/").parse().unwrap(), + certificate: Digest::from_bytes([23; 32]), + public_key: [27; 32], + }; + transport.remember_owner(target.cell_id(), peer, now + 10_000, now, Instant::now()); + assert!(matches!( + transport.send_inner(target.clone(), vec![1], 1_000).await, + Err(CellError::PeerTransportUnknown { .. }) + )); + assert_eq!(hits.load(Ordering::Relaxed), 1); + assert_eq!(counted.counts().body_requests(), 0); + assert!( + transport + .owners + .lock() + .unwrap() + .get(&target.cell_id()) + .unwrap() + .owner + .is_none() + ); + server.abort(); +} + +#[tokio::test] +async fn cancelled_send_releases_owner_hint_for_next_request() { + let started = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let router = Router::new().route( + "/internal/cells/v1/forward", + post({ + let started = Arc::clone(&started); + let release = Arc::clone(&release); + move || { + let started = Arc::clone(&started); + let release = Arc::clone(&release); + async move { + started.notify_one(); + release.notified().await; + Response::new(Body::empty()) + } + } + }), + ); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + let (transport, target, counted) = owner_lookup_fixture().await; + let now = now_ms().unwrap(); + let owner = RemotePeer { + session: SessionId::from_bytes([26; 16]), + endpoint: format!("http://{address}/").parse().unwrap(), + certificate: Digest::from_bytes([23; 32]), + public_key: [27; 32], + }; + transport.remember_owner(target.cell_id(), owner, now + 10_000, now, Instant::now()); + let sender = transport.clone(); + let request_target = target.clone(); + let request = + tokio::spawn(async move { sender.send_inner(request_target, vec![1], 5_000).await }); + started.notified().await; + request.abort(); + assert!(request.await.unwrap_err().is_cancelled()); + assert_eq!( + transport.owner(&target).await.unwrap().session, + SessionId::from_bytes([26; 16]) + ); + assert_eq!(counted.counts().body_requests(), 0); + release.notify_one(); + server.abort(); +} + +#[tokio::test] +async fn not_started_refusal_refreshes_authority_without_resending_to_stale_peer() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + let hits = Arc::new(AtomicUsize::new(0)); + let server_hits = Arc::clone(&hits); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let router = Router::new().route( + "/internal/cells/v1/forward", + post(move || { + server_hits.fetch_add(1, Ordering::Relaxed); + async { + Response::builder() + .status(StatusCode::SERVICE_UNAVAILABLE) + .body(Body::empty()) + .unwrap() + } + }), + ); + let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() }); + let (transport, target, counted) = owner_lookup_fixture().await; + let now = now_ms().unwrap(); + let peer = RemotePeer { + session: SessionId::from_bytes([26; 16]), + endpoint: format!("http://{address}/").parse().unwrap(), + certificate: Digest::from_bytes([23; 32]), + public_key: [27; 32], + }; + transport.remember_owner(target.cell_id(), peer, now + 10_000, now, Instant::now()); + assert!( + transport + .send_inner(target.clone(), vec![1], 1_000) + .await + .is_err() + ); + assert_eq!(hits.load(Ordering::Relaxed), 1); + assert_eq!(counted.counts().body_requests(), 2); + server.abort(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore = "manual adapter owner-lookup baseline"] +async fn owner_lookup_performance() { + use std::time::Instant; + + let mut raw = String::from("lane\tconcurrency\telapsed_us\n"); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let certificate_dir = std::env::temp_dir().join(format!( + "cellule-peer-bench-cert-{}-{}", + std::process::id(), + now_ms().unwrap() + )); + std::fs::create_dir(&certificate_dir).unwrap(); + let openssl = |args: &[&str]| { + let output = std::process::Command::new("openssl") + .args(args) + .current_dir(&certificate_dir) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + }; + openssl(&[ + "req", + "-x509", + "-newkey", + "ed25519", + "-nodes", + "-keyout", + "ca.key", + "-out", + "ca.crt", + "-subj", + "/CN=Cellule benchmark CA", + "-days", + "1", + "-addext", + "basicConstraints=critical,CA:TRUE", + "-addext", + "keyUsage=critical,keyCertSign,cRLSign", + ]); + openssl(&[ + "req", + "-new", + "-newkey", + "ed25519", + "-nodes", + "-keyout", + "leaf.key", + "-out", + "leaf.csr", + "-subj", + "/CN=localhost", + ]); + std::fs::write(certificate_dir.join("leaf.ext"), "subjectAltName=DNS:localhost\nextendedKeyUsage=serverAuth,clientAuth\nkeyUsage=digitalSignature\n").unwrap(); + openssl(&[ + "x509", + "-req", + "-in", + "leaf.csr", + "-CA", + "ca.crt", + "-CAkey", + "ca.key", + "-CAcreateserial", + "-out", + "leaf.crt", + "-days", + "1", + "-extfile", + "leaf.ext", + ]); + let tls = LoadedPeerTls::load( + &certificate_dir.join("leaf.crt"), + &certificate_dir.join("leaf.key"), + &certificate_dir.join("ca.crt"), + "localhost", + ) + .unwrap(); + let (transport, target, counted) = owner_lookup_fixture_with_endpoint( + format!("https://localhost:{}/", address.port()), + tls.certificate(), + tls.fleet(), + tls.signing_key().clone(), + Arc::new(tls.client_identity()), + ) + .await; + let response = cellule_runtime::peer::encode_peer_reply(&peer_wire::PeerReply { + outcome: Some(peer_wire::peer_reply::Outcome::Read(peer_wire::ReadReply { + receipt: None, + result: Some(peer_wire::read_reply::Result::Description( + peer_wire::CellDescription { + cell_id: target.cell_id().as_bytes().to_vec(), + incarnation: vec![28; 16], + code: vec![23; 32], + schema: 1, + }, + )), + })), + }) + .unwrap(); + let response = Arc::new(response); + let router = Router::new().route( + "/internal/cells/v1/forward", + post(move || { + let response = Arc::clone(&response); + async move { + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, PROTOBUF_MEDIA_TYPE) + .header(header::CACHE_CONTROL, "no-store") + .body(Body::from(response.as_ref().clone())) + .unwrap() + } + }), + ); + let tls_listener = tls.listener(listener); + let server = tokio::spawn(async move { axum::serve(tls_listener, router).await.unwrap() }); + let peer = Arc::new(RemotePeer { + session: SessionId::from_bytes([26; 16]), + endpoint: format!("https://localhost:{}/", address.port()) + .parse() + .unwrap(), + certificate: tls.certificate(), + public_key: tls.signing_key().verifying_key().to_bytes(), + }); + let cold_before = counted.counts().body_requests(); + let cold_started = Instant::now(); + transport + .send_inner(target.clone(), vec![1], 5_000) + .await + .unwrap(); + let cold_us = cold_started.elapsed().as_micros(); + let cold_reads = counted.counts().body_requests() - cold_before; + raw.push_str(&format!("cold_adapter\t1\t{cold_us}\n")); + println!("PERF cold_adapter elapsed_us={cold_us} reads={cold_reads}"); + for concurrency in [1_usize, 16] { + let before = counted.counts().body_requests(); + let started = Instant::now(); + let mut samples = Vec::with_capacity(1_024); + for _ in 0..(1_024 / concurrency) { + let calls = (0..concurrency).map(|_| { + let transport = transport.clone(); + let target = target.clone(); + async move { + let call = Instant::now(); + transport.owner(&target).await.unwrap(); + call.elapsed().as_micros() + } + }); + samples.extend(futures_util::future::join_all(calls).await); + } + let total = started.elapsed(); + let reads = counted.counts().body_requests() - before; + assert_eq!(samples.len(), 1_024); + assert!(reads <= 2 * samples.len()); + samples.sort_unstable(); + for sample in &samples { + raw.push_str(&format!("owner_lookup\t{concurrency}\t{sample}\n")); + } + println!( + "PERF owner_lookup concurrency={concurrency} calls={} elapsed_ms={:.3} throughput_per_s={:.1} p50_us={} p95_us={} p99_us={} reads={reads}", + samples.len(), + total.as_secs_f64() * 1_000.0, + samples.len() as f64 / total.as_secs_f64(), + samples[samples.len() / 2], + samples[samples.len() * 95 / 100], + samples[samples.len() * 99 / 100], + ); + let started = Instant::now(); + let mut network_samples = Vec::with_capacity(1_024); + for _ in 0..(1_024 / concurrency) { + let calls = (0..concurrency).map(|_| { + let transport = transport.clone(); + let peer = Arc::clone(&peer); + async move { + let call = Instant::now(); + assert!(matches!( + transport.send_once(&peer, vec![1], 5_000).await.unwrap(), + PeerHttpAttempt::Reply(_) + )); + call.elapsed().as_micros() + } + }); + network_samples.extend(futures_util::future::join_all(calls).await); + } + let network_total = started.elapsed(); + network_samples.sort_unstable(); + for sample in &network_samples { + raw.push_str(&format!("peer_http\t{concurrency}\t{sample}\n")); + } + println!( + "PERF peer_http concurrency={concurrency} calls={} elapsed_ms={:.3} throughput_per_s={:.1} p50_us={} p95_us={} p99_us={}", + network_samples.len(), + network_total.as_secs_f64() * 1_000.0, + network_samples.len() as f64 / network_total.as_secs_f64(), + network_samples[network_samples.len() / 2], + network_samples[network_samples.len() * 95 / 100], + network_samples[network_samples.len() * 99 / 100], + ); + // Refresh an expired observation outside the warm lane. Every sample + // below begins with the same live owner hint in the candidate. + transport.owner(&target).await.unwrap(); + let reads_before = counted.counts().body_requests(); + let started = Instant::now(); + let mut full_samples = Vec::with_capacity(1_024); + for _ in 0..(1_024 / concurrency) { + let calls = (0..concurrency).map(|_| { + let transport = transport.clone(); + let target = target.clone(); + async move { + let call = Instant::now(); + transport.send_inner(target, vec![1], 5_000).await.unwrap(); + call.elapsed().as_micros() + } + }); + full_samples.extend(futures_util::future::join_all(calls).await); + } + let full_total = started.elapsed(); + let reads = counted.counts().body_requests() - reads_before; + full_samples.sort_unstable(); + assert_eq!(full_samples.len(), 1_024); + for sample in &full_samples { + raw.push_str(&format!("full_adapter\t{concurrency}\t{sample}\n")); + } + println!( + "PERF full_adapter concurrency={concurrency} calls={} elapsed_ms={:.3} throughput_per_s={:.1} p50_us={} p95_us={} p99_us={} reads={reads}", + full_samples.len(), + full_total.as_secs_f64() * 1_000.0, + full_samples.len() as f64 / full_total.as_secs_f64(), + full_samples[full_samples.len() / 2], + full_samples[full_samples.len() * 95 / 100], + full_samples[full_samples.len() * 99 / 100], + ); + } + server.abort(); + let report = std::env::temp_dir().join(format!( + "cellule-peer-owner-lookup-{}-{}.tsv", + std::process::id(), + now_ms().unwrap() + )); + std::fs::write(&report, raw).unwrap(); + println!("PERF raw={}", report.display()); + std::fs::remove_dir_all(certificate_dir).unwrap(); +} diff --git a/crates/cellule-runtime/Cargo.toml b/crates/cellule-runtime/Cargo.toml index 2ebce6e..1fe0f70 100644 --- a/crates/cellule-runtime/Cargo.toml +++ b/crates/cellule-runtime/Cargo.toml @@ -43,5 +43,6 @@ protoc-bin-vendored = "3" [dev-dependencies] async-trait.workspace = true +cellule-store = { workspace = true, features = ["test-support"] } proptest = "1" tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/crates/cellule-runtime/docs/overview.md b/crates/cellule-runtime/docs/overview.md index b62b3ef..290a6bf 100644 --- a/crates/cellule-runtime/docs/overview.md +++ b/crates/cellule-runtime/docs/overview.md @@ -40,8 +40,8 @@ same registry, actor, SQLite, and LTX publication path. | Client constructor | Local Cell | Cell owned by another node | | --- | --- | --- | -| `CellClient::local_runtime` | Resolves the newly admitted local Cell through its catalog and owner record for each call. | Not supported; embedded single-node routing only. | -| `CellClient::runtime_with_peer` | Uses the same local path. | One authenticated peer round trip. | +| `CellClient::local_runtime` | Reads catalog and owner records in parallel for each call. | Not supported; embedded single-node routing only. | +| `CellClient::runtime_with_peer` | Checks actor admission, then verifies catalog and owner records. | A local actor miss delegates without those metadata reads to one authenticated peer round trip. | The product server supplies the peer transport and owner lookup. Neither constructor acquires an idle Cell. diff --git a/crates/cellule-runtime/src/cell/actor/acquire.rs b/crates/cellule-runtime/src/cell/actor/acquire.rs index b8b3c79..16c13bc 100644 --- a/crates/cellule-runtime/src/cell/actor/acquire.rs +++ b/crates/cellule-runtime/src/cell/actor/acquire.rs @@ -7,6 +7,27 @@ use super::*; impl CellRuntime { + /// Checks actor-owned admission before a peer route reads Cell metadata. + /// + /// A miss only means this process has no currently dispatchable handle; + /// the peer transport remains responsible for resolving remote authority. + pub(crate) async fn has_local_owner(&self, cell: CellId) -> crate::Result { + self.ensure_running()?; + let (reply, response) = oneshot::channel(); + self.inner + .sender + .send(Message::Lookup { + cell, + require_resident: false, + reply, + }) + .await + .map_err(|_| Error::RuntimeClosed)?; + let local = response.await.map_err(|_| Error::RuntimeClosed)?; + self.ensure_running()?; + Ok(local.is_some()) + } + /// Resolves an active local owner without exposing the dispatcher's Cell map. pub async fn local_handle( &self, diff --git a/crates/cellule-runtime/src/cell/actor/requests.rs b/crates/cellule-runtime/src/cell/actor/requests.rs index 41c3e3a..7e9fc26 100644 --- a/crates/cellule-runtime/src/cell/actor/requests.rs +++ b/crates/cellule-runtime/src/cell/actor/requests.rs @@ -146,11 +146,12 @@ pub(super) async fn execute_command( effect_id: u64, ) -> TaskResult { let execution_started = std::time::Instant::now(); + let queue_wait = command.queued_at.elapsed(); tracing::debug!( target: "cellule_runtime::action", parent: &command.trace, event = "cell_execution_started", - actor_queue_us = command.queued_at.elapsed().as_micros(), + actor_queue_us = queue_wait.as_micros(), ); let deadline = SqlDeadline::new(std::time::Instant::now() + SQL_WALL_DEADLINE); let execution = match command.handler.take() { @@ -198,6 +199,11 @@ pub(super) async fn execute_command( match tokio::time::timeout_at(deadline.at().into(), &mut operation).await { Ok(result) => result, Err(_) => { + command.telemetry.command_execution( + queue_wait, + execution_started.elapsed(), + false, + ); let fenced = !deadline.cancel_queued(); tracing::warn!(cell = ?command.cell, sql_started = fenced, "Cell SQL command deadline expired"); if fenced { @@ -227,6 +233,9 @@ pub(super) async fn execute_command( } None => Err(Error::Fenced), }; + command + .telemetry + .command_execution(queue_wait, execution_started.elapsed(), execution.is_ok()); tracing::debug!( target: "cellule_runtime::action", parent: &command.trace, @@ -371,6 +380,8 @@ pub(super) fn start_publication( pool: &SqlWorkerPool, tasks: &mut JoinSet, ) { + use crate::fleet::telemetry::PublicationTiming; + let Some(mut publisher) = active.publisher.take() else { return; }; @@ -383,6 +394,7 @@ pub(super) fn start_publication( - i128::from(active.published_sequence); let incarnation = active.incarnation; let commit_sequence = publication.pending.outcome().commit_sequence(); + let queue_wait = publication.submitted_at.elapsed(); tracing::debug!( target: "cellule_runtime::action", event = "cell_publication_started", @@ -409,8 +421,11 @@ pub(super) fn start_publication( let mut publication_proof = Some(publication.proof); let fleet_deadline = std::time::Instant::now() + FLEET_PUBLICATION_GRACE; let mut retry_delay = std::time::Duration::from_millis(100); + let mut preparation = std::time::Duration::ZERO; + let mut authority = std::time::Duration::ZERO; let result = async { let expected = publication.pending.outcome().clone(); + let preparation_started = std::time::Instant::now(); let prepared = loop { match publisher.prepare(&publication.pending).await { Ok(prepared) => break prepared, @@ -432,7 +447,9 @@ pub(super) fn start_publication( Err(error) => return Err(error), } }; + preparation = preparation_started.elapsed(); pool.bind_prepared(cell, prepared.clone()).await?; + let authority_started = std::time::Instant::now(); let root = loop { match publisher .publish_prepared(&prepared, publication.pending.next_due_ms()) @@ -457,6 +474,7 @@ pub(super) fn start_publication( Err(error) => return Err(error), } }; + authority = authority_started.elapsed(); if let Some(durability) = publication.durability.as_ref() { durability.prove_object().await?; } else { @@ -471,6 +489,14 @@ pub(super) fn start_publication( Ok(()) } .await; + publisher.record_publication_timing(PublicationTiming { + queue_wait, + preparation, + authority, + total: publication.submitted_at.elapsed(), + succeeded: result.is_ok(), + commit_sequence, + }); tracing::debug!( target: "cellule_runtime::action", event = "cell_publication_completed", diff --git a/crates/cellule-runtime/src/client/mod.rs b/crates/cellule-runtime/src/client/mod.rs index 3475164..d19ecf9 100644 --- a/crates/cellule-runtime/src/client/mod.rs +++ b/crates/cellule-runtime/src/client/mod.rs @@ -724,9 +724,9 @@ impl CellClient { /// Routes a target to its current local owner or an authenticated peer. /// - /// Every invocation rechecks catalog and authority state. The peer round - /// trip must resolve the current remote owner and verify its enrollment; - /// this constructor does not acquire an idle Cell. + /// A local actor hit rechecks catalog and authority state. A local miss + /// delegates without those reads; the peer round trip resolves the remote + /// owner and verifies enrollment. This does not acquire an idle Cell. #[must_use] pub fn runtime_with_peer( registry: Arc, @@ -736,8 +736,13 @@ impl CellClient { principal: crate::peer::PeerPrincipal, round_trip: Arc, ) -> Self { - Self::peer(registry, signer, principal, round_trip) - .with_local_resolver(Arc::new(RuntimeLocalResolver { runtime, layout })) + Self::peer(registry, signer, principal, round_trip).with_local_resolver(Arc::new( + RuntimeLocalResolver { + runtime, + layout, + remote_on_miss: true, + }, + )) } /// Resolves a local owner before delegating to this client's transport. diff --git a/crates/cellule-runtime/src/client/runtime.rs b/crates/cellule-runtime/src/client/runtime.rs index 788c677..5605d4b 100644 --- a/crates/cellule-runtime/src/client/runtime.rs +++ b/crates/cellule-runtime/src/client/runtime.rs @@ -33,7 +33,11 @@ impl RuntimeCellTransport { ) -> Self { Self { registry, - resolver: Arc::new(RuntimeLocalResolver { runtime, layout }), + resolver: Arc::new(RuntimeLocalResolver { + runtime, + layout, + remote_on_miss: false, + }), remote: None, } } @@ -71,6 +75,7 @@ impl RuntimeCellTransport { pub(super) struct RuntimeLocalResolver { pub(super) runtime: CellRuntime, pub(super) layout: CellStorageLayout, + pub(super) remote_on_miss: bool, } impl LocalCellResolver for RuntimeLocalResolver { @@ -80,14 +85,21 @@ impl LocalCellResolver for RuntimeLocalResolver { ) -> Pin>> + Send + 'static>> { let resolver = self.clone(); Box::pin(async move { - let catalog = CellCatalog::new(resolver.layout.clone(), target.tenant()) - .lookup(target.cell_id()) - .await? - .ok_or(Error::Control("target Cell is not cataloged"))?; - let control = CellAuthority::new(resolver.layout) - .load(target.cell_id()) - .await? - .ok_or(Error::Control("target Cell has no authority record"))?; + if resolver.remote_on_miss + && !resolver.runtime.has_local_owner(target.cell_id()).await? + { + // No local actor can execute this request. The peer route + // resolves remote ownership and its receiver fences stale hints. + return Ok(None); + } + let catalog = CellCatalog::new(resolver.layout.clone(), target.tenant()); + let authority = CellAuthority::new(resolver.layout); + let (catalog, control) = tokio::join!( + catalog.lookup(target.cell_id()), + authority.load(target.cell_id()) + ); + let catalog = catalog?.ok_or(Error::Control("target Cell is not cataloged"))?; + let control = control?.ok_or(Error::Control("target Cell has no authority record"))?; resolver.runtime.local_handle(catalog, &control).await }) } diff --git a/crates/cellule-runtime/src/fleet/telemetry.rs b/crates/cellule-runtime/src/fleet/telemetry.rs index 3d7106d..4541409 100644 --- a/crates/cellule-runtime/src/fleet/telemetry.rs +++ b/crates/cellule-runtime/src/fleet/telemetry.rs @@ -1,6 +1,7 @@ //! Bounded operational telemetry emitted by the runtime. use std::{sync::Arc, time::Duration}; +use crate::CellId; use crate::fleet::pressure::PressureState; use crate::node::log::DurabilitySource; @@ -15,6 +16,24 @@ pub enum CommandResponseSource { Object, } +/// One completed object publication, which may finish after a follower-proof +/// response has already been released for the same commit sequence. +#[derive(Clone, Copy, Debug)] +pub struct PublicationTiming { + /// Time spent queued behind earlier roots for this Cell. + pub queue_wait: Duration, + /// Time spent preparing the immutable root, including bounded retries. + pub preparation: Duration, + /// Time spent publishing the prepared root through authority CAS. + pub authority: Duration, + /// Time from queued publication to terminal completion. + pub total: Duration, + /// Whether object publication completed and the worker confirmed the root. + pub succeeded: bool, + /// Sequence used to correlate this observation with a request trace. + pub commit_sequence: u64, +} + /// Outcome of an actor-owned resident route lookup. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum ResidentRouteOutcome { @@ -135,6 +154,20 @@ pub trait CellTelemetry: Send + Sync { ) { } + /// Records the actor queue wait and SQL worker round trip for one command. + /// The outcome describes worker execution, before durability proof. + fn command_execution( + &self, + _queue_wait: Duration, + _worker_round_trip: Duration, + _succeeded: bool, + ) { + } + + /// Records background root progress separately from the response winner. + /// Cell IDs and sequences are for local trace correlation, never metric labels. + fn publication_completed(&self, _cell: CellId, _timing: PublicationTiming) {} + /// Records how one commit's node-log submission resolved. fn durability_submission(&self, _outcome: DurabilitySubmissionOutcome) {} @@ -235,6 +268,23 @@ impl CellTelemetryHandle { } } + pub(crate) fn command_execution( + &self, + queue_wait: Duration, + worker_round_trip: Duration, + succeeded: bool, + ) { + if let Some(telemetry) = self.inner.get() { + telemetry.command_execution(queue_wait, worker_round_trip, succeeded); + } + } + + pub(crate) fn publication_completed(&self, cell: CellId, timing: PublicationTiming) { + if let Some(telemetry) = self.inner.get() { + telemetry.publication_completed(cell, timing); + } + } + pub(crate) fn catalog_read(&self, kind: CatalogReadKind, elapsed: Duration, succeeded: bool) { if let Some(telemetry) = self.inner.get() { telemetry.catalog_read(kind, elapsed, succeeded); diff --git a/crates/cellule-runtime/src/publication/mod.rs b/crates/cellule-runtime/src/publication/mod.rs index c729ef8..2791f47 100644 --- a/crates/cellule-runtime/src/publication/mod.rs +++ b/crates/cellule-runtime/src/publication/mod.rs @@ -2,7 +2,7 @@ use crate::cell::executor::{CellExecutor, StoredOutcome}; use crate::control::Transition; use crate::control::authority::{CellAuthority, VersionedControl}; -use crate::fleet::telemetry::DurabilitySubmissionOutcome; +use crate::fleet::telemetry::{DurabilitySubmissionOutcome, PublicationTiming}; use crate::identity::{ApplicationId, encode_hex}; use crate::node::durability::NodeDurability; use crate::node::log::CommitTicket; @@ -117,6 +117,11 @@ impl CellPublisher { .durability_proof(crate::node::log::DurabilitySource::Object, waited); } + pub(crate) fn record_publication_timing(&self, timing: PublicationTiming) { + self.telemetry + .publication_completed(self.observed.value().cell, timing); + } + /// Returns the control version the publisher last observed. #[must_use] pub fn control(&self) -> &VersionedControl { diff --git a/crates/cellule-runtime/tests/protocol/client.rs b/crates/cellule-runtime/tests/protocol/client.rs index b4d4025..40febd8 100644 --- a/crates/cellule-runtime/tests/protocol/client.rs +++ b/crates/cellule-runtime/tests/protocol/client.rs @@ -453,6 +453,10 @@ async fn fixture() -> Fixture { } async fn fixture_with_limits(limits: Limits) -> Fixture { + fixture_with_store(limits, Store::new(Arc::new(InMemory::new()))).await +} + +async fn fixture_with_store(limits: Limits, store: Store) -> Fixture { let registry = registry(); let target = CellTarget::new( TenantId::from_bytes([1; 16]), @@ -463,7 +467,6 @@ async fn fixture_with_limits(limits: Limits) -> Fixture { .unwrap(); let cell = target.cell_id(); let incarnation = IncarnationId::from_bytes([4; 16]); - let store = Store::new(Arc::new(InMemory::new())); let layout = CellStorageLayout::new(store, Path::from("client"), [2; 16]); let replica = CellReplica::new( layout.clone(), diff --git a/crates/cellule-runtime/tests/protocol/client/typed.rs b/crates/cellule-runtime/tests/protocol/client/typed.rs index 9e73fbe..5966484 100644 --- a/crates/cellule-runtime/tests/protocol/client/typed.rs +++ b/crates/cellule-runtime/tests/protocol/client/typed.rs @@ -2,6 +2,7 @@ use super::*; use cellule_runtime::cell::actor::CellHandle; +use cellule_store::test_support::CountingObjectStore; #[tokio::test] async fn local_resolver_refusal_never_dispatches_to_the_underlying_owner() { @@ -576,6 +577,101 @@ async fn runtime_client_forwards_to_the_remote_owner() { caller.shutdown().await.unwrap(); fixture.handle().drain().await.unwrap(); } + +#[tokio::test] +async fn remote_runtime_route_skips_local_metadata_without_skipping_local_owner_validation() { + let counted = Arc::new(CountingObjectStore::new(Arc::new(InMemory::new()))); + let fixture = fixture_with_store(Limits::default(), Store::new(counted.clone())).await; + let caller = CellRuntime::new( + SqlWorkerPool::new(1, 4).unwrap(), + 4 * 1024 * 1024, + SessionId::from_bytes([64; 16]), + ) + .unwrap(); + let signer = Arc::new(PeerSigner::new( + SessionId::from_bytes([65; 16]), + fixture.registry.release_digest(), + ed25519_dalek::SigningKey::from_bytes(&[66; 32]), + )); + let round_trip: Arc = Arc::new(LoopbackRoundTrip { + verifier: Arc::new(PeerVerifier::new( + SessionId::from_bytes([65; 16]), + fixture.registry.release_digest(), + signer.verifying_key(), + )), + dispatcher: Arc::new(PeerDispatcher::new( + Arc::clone(&fixture.registry), + Arc::new(LocalResolver { + target: fixture.target.clone(), + handle: fixture.handle().clone(), + }), + Arc::new(RepositoryAuthorizer), + )), + }); + let description = CellDescription { + cell: fixture.target.cell_id(), + incarnation: fixture.incarnation, + code: fixture.registry.module_code(MODULE).unwrap(), + schema: 1, + }; + let principal = PeerPrincipal { + issuer: "https://identity.example".into(), + subject: "alice".into(), + actions: vec!["repository.issue.create".into()], + }; + let remote = CellClient::runtime_with_peer( + Arc::clone(&fixture.registry), + caller.clone(), + fixture.layout.clone(), + Arc::clone(&signer), + principal.clone(), + Arc::clone(&round_trip), + ); + counted.reset(); + assert_eq!( + remote + .query::(&fixture.target, None, ()) + .await + .unwrap() + .output, + 0 + ); + assert_eq!(counted.counts().body_requests(), 0); + + let remote = remote.with_observed_description(description); + counted.reset(); + assert_eq!( + remote + .query::(&fixture.target, None, ()) + .await + .unwrap() + .output, + 0 + ); + assert_eq!(counted.counts().body_requests(), 0); + + let local = CellClient::runtime_with_peer( + Arc::clone(&fixture.registry), + fixture.runtime.as_ref().unwrap().clone(), + fixture.layout.clone(), + signer, + principal, + round_trip, + ) + .with_observed_description(description); + counted.reset(); + assert_eq!( + local + .query::(&fixture.target, None, ()) + .await + .unwrap() + .output, + 0 + ); + assert_eq!(counted.counts().body_requests(), 3); + caller.shutdown().await.unwrap(); + fixture.handle().drain().await.unwrap(); +} #[tokio::test] async fn typed_client_rejects_conflicting_identity_receipt_and_module_before_execution() { let fixture = fixture().await; diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/durability.rs b/crates/cellule-runtime/tests/runtime/lifecycle/durability.rs index 1c389ec..d463533 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/durability.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/durability.rs @@ -1,14 +1,17 @@ //! Node-log durability, fleet proofs, and byte admission. use super::*; -use cellule_runtime::fleet::telemetry::{CellTelemetry, CommandResponseSource}; +use cellule_runtime::fleet::telemetry::{CellTelemetry, CommandResponseSource, PublicationTiming}; mod admission; mod proofs; mod recovery; #[derive(Default)] -pub(super) struct RecordingResponses(pub(super) Mutex>); +pub(super) struct RecordingResponses( + pub(super) Mutex>, + pub(super) Mutex>, +); impl CellTelemetry for RecordingResponses { fn command_response( @@ -23,4 +26,8 @@ impl CellTelemetry for RecordingResponses { } self.0.lock().unwrap().push(source); } + + fn publication_completed(&self, _cell: cellule_runtime::CellId, timing: PublicationTiming) { + self.1.lock().unwrap().push(timing); + } } diff --git a/crates/cellule-runtime/tests/runtime/lifecycle/durability/proofs.rs b/crates/cellule-runtime/tests/runtime/lifecycle/durability/proofs.rs index 4820d95..8356c6b 100644 --- a/crates/cellule-runtime/tests/runtime/lifecycle/durability/proofs.rs +++ b/crates/cellule-runtime/tests/runtime/lifecycle/durability/proofs.rs @@ -68,6 +68,12 @@ async fn accepted_control_cas_with_lost_response_releases_once() { CommandResponseSource::Recorded ] ); + { + let publication_timings = responses.1.lock().unwrap(); + assert_eq!(publication_timings.len(), 1); + assert_eq!(publication_timings[0].commit_sequence, 1); + assert!(publication_timings[0].succeeded); + } let restored = fixture._directory.path().join("lost-cas-restored.sqlite"); let verified = fixture.replica.open_root(&root).await.unwrap(); @@ -164,6 +170,7 @@ async fn capture_failure_after_sql_commit_fences_until_authoritative_recovery() ); runtime.shutdown().await.unwrap(); assert_eq!(responses.0.lock().unwrap().as_slice(), &[]); + assert!(responses.1.lock().unwrap().is_empty()); let restored = fixture._directory.path().join("capture-restored.sqlite"); let verified = fixture.replica.open_root(&root).await.unwrap(); @@ -847,6 +854,12 @@ async fn exercise_fleet_ack_drain(lose_response: bool) { CommandResponseSource::Recorded ] ); + { + let publication_timings = responses.1.lock().unwrap(); + assert_eq!(publication_timings.len(), 1); + assert_eq!(publication_timings[0].commit_sequence, 1); + assert!(publication_timings[0].succeeded); + } let verified = fixture.replica.open_root(&published_root).await.unwrap(); let recovered = fixture