diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69f46d0e..2841225b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,9 +92,9 @@ jobs: - name: Build and import standalone SDK wheel run: | python scripts/build_qdl_sdk_release.py --output-dir /tmp/qdl-sdk - python -m pip install --disable-pip-version-check /tmp/qdl-sdk/qdl_sdk-2.0.0-py3-none-any.whl + python -m pip install --disable-pip-version-check /tmp/qdl-sdk/qdl_sdk-2.0.1-py3-none-any.whl cd "$(mktemp -d)" - PYTHONPATH= python -c 'import qdl_sdk; assert qdl_sdk.__version__ == "2.0.0"' + PYTHONPATH= python -c 'import qdl_sdk; assert qdl_sdk.__version__ == "2.0.1"' unit-tests: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..ca0239e1 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,50 @@ +name: Publish Certified Release + +on: + push: + tags: ['v2.*'] + +permissions: + contents: write + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + fetch-depth: 0 + - name: Verify stable ancestry and certificate + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + git merge-base --is-ancestor "$GITHUB_SHA" origin/main + python3 - <<'PY' + import json + import os + from pathlib import Path + import re + tag = os.environ['RELEASE_TAG'] + if not re.fullmatch(r'v2\.\d+\.\d+', tag): + raise SystemExit('Expected a stable v2 semantic-version tag') + root = Path('upgrade/evidence/releases') / tag + if json.loads((root / 'certificate.json').read_text())['status'] != 'PASS': + raise SystemExit('Release certificate is not PASS') + if not (root / 'RELEASE_NOTES.md').is_file(): + raise SystemExit('Release notes are missing') + PY + - name: Publish immutable tag and public evidence + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.ref_name }} + run: | + evidence="upgrade/evidence/releases/$RELEASE_TAG" + gh release create "$RELEASE_TAG" --verify-tag \ + --title "Quant Data Layer $RELEASE_TAG" \ + --notes-file "$evidence/RELEASE_NOTES.md" \ + "$evidence/certificate.json" diff --git a/Cargo.lock b/Cargo.lock index fcca526e..c564ee76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -291,6 +291,20 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + [[package]] name = "glob" version = "0.3.4" @@ -533,6 +547,12 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "memchr" version = "2.8.3" @@ -787,13 +807,14 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.9" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "bytes", - "getrandom", - "rand", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.5", "ring", "rustc-hash", "rustls", @@ -828,6 +849,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "rand" version = "0.8.7" @@ -835,8 +862,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", ] [[package]] @@ -846,7 +883,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -855,7 +902,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", ] [[package]] @@ -980,7 +1036,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", @@ -1420,7 +1476,7 @@ dependencies = [ "http", "httparse", "log", - "rand", + "rand 0.8.7", "rustls", "rustls-pki-types", "sha1", @@ -1517,6 +1573,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.127" @@ -1707,6 +1772,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "zerocopy" version = "0.8.56" diff --git a/DATA_LAYER_SERVICE_ACCESS_GUIDE.md b/DATA_LAYER_SERVICE_ACCESS_GUIDE.md index 461cae52..5ada24fe 100644 --- a/DATA_LAYER_SERVICE_ACCESS_GUIDE.md +++ b/DATA_LAYER_SERVICE_ACCESS_GUIDE.md @@ -4,7 +4,52 @@ This document provides technical details on how to integrate production services ## Standard -Other services should connect to `data_layer` using this rule: +V2-routed Binance/OKX consumers use the versioned manifest and `qdl_sdk`: + +- `AsyncDataLayerClient` (or `DataLayerClientV2` for synchronous reads) calls + authenticated V2 query replicas and the signed-cursor gRPC stream service. +- Warm up once, then append/deduplicate final events into the alpha's bounded + buffer. Restore the acknowledged cursor on reconnect. REST warmup/history + repairs missing state; it is not a per-tick polling loop. +- Resolve canonical instrument identity before reading. Venue, native symbol, + feed, interval, source policy, depth and freshness must match the consumer + manifest. A provider wrapper alone does not grant an identity access. +- V1 is fallback only where the manifest explicitly permits it; `BLOCKED` + products cannot silently fall back, substitute venues or direct-connect a + provider. VN remains on its separately governed V1 route. +- Closed BARs drive signals. QUOTE/TRADE provide current price context; + verified BOOK_SNAPSHOT/BOOK_DELTA support limit-price and impact analysis. + MARK_INDEX_PRICE is used only under the declared trigger/risk policy. + Trading System, not Data Layer, owns order submission, OCO and risk admission. + +V2 query operations are: + +```text +GET /v2/instruments +GET /v2/instruments/{identity} +GET /v2/market-data/{instrument_uid}/snapshot +GET /v2/market-data/{instrument_uid}/warmup +GET /v2/market-data/{instrument_uid}/history +POST /v2/market-data/warmup:batch +POST /v2/market-data/reference:batch +GET /v2/feeds/{instrument_uid}/status +POST /v2/system/readiness:check +GET /v2/system/readiness +GET /v2/data-quality/gaps +``` + +Use deployment-provided query/stream addresses, mTLS identity and JWT config; +port8100 and the Redis channels below describe **V1 compatibility**, not the +V2 authority. `warmup_then_stream()` exposes the SDK's signed snapshot/replay/ +live handoff; no new per-symbol container or provider connection is needed. +Stale, partial or gapped execution data stays rejected. Bounded retry waits for +a new valid read; it never changes old data's timestamp or treats session +liveness as a fresh executable book/price. + +### V1 Compatibility And Fallback + +The V1-specific sections below retain these integration rules for existing +consumers and explicitly permitted fallback: - use Redis Pub/Sub for live streaming consumption - use REST API for warmup, latest-state recovery, diagnostics, and manual triggers diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 33868d9f..7fcc9179 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -28373,3 +28373,7973 @@ Remote feature branches remain intact for PR history; local still ahead of its remote by two commits. No Docker, runtime, provider, Kafka, Redis, SQLite, V1/V2 consumer, Trading System, alpha, broker or order resource was changed. + +## 24.3.18 Phase 53.3 V2 Query Readiness Prerequisite + +**Status:** `IN PROGRESS / SOURCE REPAIR BEFORE ALPHA NO-ORDER PACKET` +(2026-09-01). + +**Linkage and goal.** This is the Data Layer prerequisite for Trading System +Unified Plan Section 53.3, *Representative Alpha No-Order Proof, Paper +Rollout And Broad Config Readiness*. The owner approved that phase. Before a +real-provider alpha may consume V2, both V2 query replicas must report their +actual cache state correctly; a false `NOT_READY` cannot be bypassed through +V1 or a relaxed SLA. + +**Observed bounded diagnosis.** Both query replicas remain live with zero +restart/OOM evidence, but their required `query_cache` readiness probe invokes +`SQLiteDurableSpool.stats()`. That diagnostic performs a full `events` table +aggregate. Against the current shared canonical spool (roughly 398k records; +about 0.95 GiB database plus 2.27 GiB WAL), the aggregate exceeded the +one-second probe deadline and turned a readable cache into `NOT_READY`. +`spool_state.event_records` and `spool_state.payload_bytes` already maintain +the same capacity counters atomically on append and trim; the readiness path +does not require diagnostic oldest/newest/storage aggregation. + +**Approved source scope.** Add one provider-neutral, read-only spool readiness +summary backed by the existing singleton `spool_state` row, and make only the +stable `query_cache` health probe use it. Preserve `stats()` unchanged for +diagnostics, preserve capacity/maintenance semantics, and fail closed when +the singleton row is absent or SQLite rejects the bounded read. No provider, +contract, manifest, authority, Rust core, ingestion, projector, stream, +Redis, Kafka, SQLite data, V1 endpoint, Trading System, alpha, broker or +order-path change is part of this source slice. + +**Source gates and rollback.** Deterministic tests must prove the summary +tracks append/trim usage, does not call the full diagnostic aggregate, and +still makes readiness `NOT_READY` on a database failure. Run focused Python +unit tests, compile and `git diff --check` in the existing non-root, +network-disabled test image. Rollback is a source revert. Only after source +exit may a separately recorded, serial two-query-reader runtime packet build +one immutable Python image and recreate `query_v2_1` then `query_v2_2`; its +exact current image remains rollback. Alpha containers, Trading System and +all order paths stay untouched until the query readiness evidence passes. + +**Source implementation and exit (`PASS / RUNTIME PACKET NEXT`, 2026-09-01).** +`SQLiteDurableSpool.readiness_summary()` now reads exactly the existing +singleton `spool_state` usage counters under the spool lock and rejects a +missing/negative state with `PayloadCorruption`. `SpoolStats` and its full +aggregate remain unchanged. Stable query readiness now calls only that bounded +summary and reports readable record/payload counters; a raised SQLite error is +still converted by the shared readiness framework into required `NOT_READY`. + +**Evidence actually run:** + +- Network-disabled/read-only `qdl-v2-python:2.0.0-7c29542` tests: + `tests.test_phase533_query_readiness` plus + `tests.test_phaseb_stable_release`: **10/10 passed**. +- The same isolated runner executed `tests.test_phaseb_stable_edge` plus + `tests.test_phase115c_five_liquid_handoff`: **56 passed, 1 intentionally + skipped** (`isolated Redis is not configured`). This covers stable catalog + identity/lineage, final-bar/query contracts, cursor replay, quiet/disconnect + session semantics, projector recovery, L2 contract and five-symbol OKX + status isolation. +- Host `python3 -B -m compileall` over each changed module and test, and + `git diff --check`, passed. The retained `tradingsystem-test:latest` image + has an obsolete Ruff parser which rejects this repository's already-existing + Poetry caret specifier before inspecting source; no dependency was installed + and no lint result is claimed from that incompatible tool. + +**Runtime / cleanup boundary.** Every test container used `--rm`, network +`none`, read-only source and tmpfs `/tmp`; no image, cache, service, Kafka, +Redis, SQLite, provider, V1/V2 route, Trading System, alpha or order state was +created or changed. The next permitted operation is one immutable Python +reader image followed by serial recreate of only `query_v2_1` and +`query_v2_2`, retaining their exact current image as rollback; it must then +prove mTLS readiness before the alpha no-order packet begins. + +**Bounded reader runtime packet (`APPROVED / EXECUTING`, 2026-09-01).** Build +exactly one immutable Python image from source commit `9d42d69` as +`qdl-v2-python:2.0.0-phase533-readiness-9d42d69`, with the commit and release +OCI build arguments. Validate the existing stable Compose topology using the +same non-secret runtime bundle already mounted by the readers: +`/home/bobby/.local/state/qdl-v2/session-liveness-43cdbe3-20260829T162719Z/phase12-paper-20260830/phase12.runtime.env`, plus only the four currently +active sealed Compose overlays recorded in the running container labels. The +active final overlay intentionally pins the old reader image, so a final +payload-free override applies the new immutable image only to `query_v2_1` +and `query_v2_2`; it follows rather than alters the sealed overlays. Compose +then targets `docker compose up -d --no-deps query_v2_1`, followed by the same +command for `query_v2_2`. + +**Exact blast radius and rollback.** Both roles retain their existing command, +memory `512 MiB`, CPU `0.75`, read-only filesystem, `stable_state`, +`stable_tls` and `/runtime` mounts. The direct rollback coordinate for each is +its current image +`sha256:7b5c848d2add9e0e36d88bfb837c93448140b7ade3a87d9e94e4902b2bf3f18e` +with the same bundle/overlays and only that target role recreated. This packet +does not reset Kafka offsets/topology, flush Redis, delete/checkpoint SQLite, +or recreate V1, Rust core, ingestors, bar edge, projectors, streams, Trading +System, alpha or the order path. It performs normal process startup reads +only; no acceptance client, provider request or consumer route is started. + +**Runtime exit.** After each serial recreate, require mTLS `/health/ready` +`200`, all required dependencies `READY`, restart count `0`, `OOMKilled=false` +and a bounded error scan. A repeated three-sample readiness check must pass +for both replicas before any alpha no-order packet is eligible. Retain exactly +the new active reader image and the named prior rollback image; no image/cache +prune occurs in this packet. + +**In-packet startup diagnosis and corrected source scope (2026-09-01).** The +first serial `query_v2_1` recreate used the new bounded health probe but did +not bind its port: process state was uninterruptible SQLite I/O with the +canonical database/WAL open, no crash and no OOM. Read-only code inspection +identified the second full scan: `_migrate()` executes `INSERT OR IGNORE INTO +spool_state SELECT COUNT(*), SUM(LENGTH(payload)) FROM events` on every open, +even when the singleton state row already exists. The role was immediately +recreated back on its sealed rollback image; `query_v2_2` and every excluded +role remained untouched. + +The same provider-neutral repair now also makes initialization check the +existing singleton first. Only a genuinely legacy/missing state row enters a +short SQLite `BEGIN IMMEDIATE` initialization transaction and performs the +one necessary aggregate to reconstruct exact capacity. Existing state opens +must make no `events` aggregate; concurrent initializers serialize and retain +the same correct result. Regression must cover existing-state no-aggregate, +missing-state reconstruction, append/trim counters and fail-closed readiness. +No retention, event ID, cursor, provider, manifest, Rust, Kafka, Redis, V1, +or external consumer behavior changes. The superseded image is not retained +as a rollout candidate; rebuild one replacement only after this source exit. + +**Startup repair source exit (`PASS / REPLACEMENT READER NEXT`, 2026-09-01).** +The migration now creates schema objects first, checks `spool_state` by its +singleton key, and returns immediately for a live cache. A missing state is +reconstructed inside `BEGIN IMMEDIATE`, re-checking after the lock so only one +initializer can aggregate legacy events; rollback on failure preserves the +previous durable state. The former source commit's bounded health summary is +retained and now has a startup path with the same bounded behavior. + +**Evidence actually run:** host `compileall` and `git diff --check` passed; +the existing non-root, network-disabled/read-only Python runner executed the +focused stable suite **68 passed, 1 intentionally skipped** (the only skipped +case needs an explicitly isolated Redis). The five new/updated checks cover +existing-state reopen without aggregate, missing-state exact reconstruction, +append/trim counter agreement, no `stats()` call in health, and fail-closed +SQLite error. Existing catalog, query, replay, projector recovery, session, +L2 and five-symbol OKX identity tests remain green. + +**First image/role cleanup evidence.** The first candidate image reached only +the source-open path on `query_v2_1`, exposed no listener and made no provider +or consumer request. Its pre-bind scan was diagnosed, then exactly that role +was recreated back to its prior sealed image. `query_v2_2` and all excluded +roles were never recreated. The first candidate image is now an unreferenced +test artifact and will be removed only after the replacement image passes +runtime readiness; no cache, volume, Kafka, Redis or data-plane cleanup is +authorized. + +**Replacement reader packet (`APPROVED / EXECUTING`, 2026-09-01).** The exact +replacement source is commit `de4aca8`; build it once as +`qdl-v2-python:2.0.0-phase533-readiness-de4aca8`. The final temporary two-role +override is updated to that tag and follows all same sealed overlays/bundle. +It serially recreates only `query_v2_1` and `query_v2_2`, preserving the +previous `sha256:7b5c…f3f18e` image as the direct rollback coordinate. The +failed first candidate `qdl-v2-python:2.0.0-phase533-readiness-9d42d69` must +not be used again and is removed only after this replacement passes. + +**Second startup scan diagnosis and final source scope (2026-09-01).** While +the replacement `query_v2_1` bound and returned all-ready mTLS health, +`query_v2_2` exposed a second independent open-path scan: default +`SQLiteDurableSpool.__init__()` runs `PRAGMA quick_check(1)` after every +schema open. It creates SQLite temporary I/O and scans the shared cache/WAL +before a reader can bind. This is valuable as an explicit integrity audit but +is not a safe per-process liveness prerequisite for the rebuildable stable +cache. + +The final same-scope correction adds a `SpoolConfig` open-integrity flag that +remains **true by default** for all other spool uses. Only +`build_stable_spool()` selects `false`, leaving the existing explicit +`integrity_check()` API/runbook intact. Stable startup instead proves readable +durable state through the bounded `spool_state` query; any SQLite error still +fails required readiness closed. Regression must prove default integrity +behavior remains, stable configuration skips only the open-time full scan, +and all prior startup/health cases remain correct. No cache data, provider, +event/cursor semantics, Kafka, Redis, V1, Rust or consumer policy changes. + +**Final startup source exit (`PASS / FINAL READER ROLL NEXT`, 2026-09-01).** +`SpoolConfig.verify_integrity_on_open` now defaults to `true`; ordinary durable +spools therefore retain the prior corruption check. Only the rebuildable +stable canonical cache opts out at process-open time. Its explicit +`integrity_check()` remains available for a deliberate maintenance audit, and +the normal V2 reader liveness path remains fail-closed through the bounded +state-row read. This removes the last known whole-cache scan from query-reader +startup without weakening default spool integrity semantics. + +**Evidence actually run:** host `compileall` and `git diff --check` passed. +The existing non-root, network-disabled, read-only Python image ran +`tests.test_phase533_query_readiness`, `tests.test_phaseb_stable_release`, +`tests.test_phaseb_stable_edge`, and +`tests.test_phase115c_five_liquid_handoff`: **70 passed, 1 intentionally +skipped** (the isolated-Redis case). New regression checks prove ordinary +spools still execute their open integrity check, stable cache construction +does not, and a manual stable-cache integrity audit remains available. Earlier +tests retain exact legacy-state reconstruction, append/trim accounting and +fail-closed bounded readiness coverage. + +**Next bounded runtime step.** Build exactly one replacement immutable reader +image from this final source commit, update the existing two-role temporary +override, and serially recreate only `query_v2_1` then `query_v2_2`. Require +three all-ready mTLS samples, no restart/OOM and no bounded errors on each role +before removing only the two unreferenced failed candidate images. No alpha +or Trading System proof begins until this exit passes. + +**Last startup-path correction (`PASS / FINAL IMAGE REBUILD`, 2026-09-01).** +Live `query_v2_1` inspection showed its process still blocked in SQLite I/O +after the two bounded-state repairs. The remaining cause was the original +schema script's unconditional `DROP INDEX` followed by recreate of the +retention index on every spool open. It is neither a migration nor a +correctness requirement when the named index already exists, and it forces a +whole-`events` reindex against the shared cache. The schema now keeps only +`CREATE INDEX IF NOT EXISTS`; a normal open preserves the existing index. +There is no event, cursor, retention, provider, Kafka, Redis, V1 or contract +semantic change. + +**Evidence actually run:** `compileall` and `git diff --check` passed. The +same isolated non-root/read-only/no-network suite now reports **71 passed, 1 +intentionally skipped**. Its new trace-level regression proves reopening a +spool requests the retention index declaratively but does not issue `DROP +INDEX`; previous tests continue to cover normal integrity checking, stable +cache open behavior, legacy state reconstruction and fail-closed readiness. +The final image must be rebuilt from this commit; the two previous candidate +images remain non-authoritative and are removed only after final runtime exit. + +**Shared-runtime impact and bounded completion packet (`APPROVED / EXECUTING`, +2026-09-01).** Runtime inspection after the reader repair found that +`stream_v2_active` and `stream_v2_passive` still ran the old image. Both build +the same stable spool and consequently reported `query_cache=NOT_READY` from +the same obsolete full-scan readiness path. The three stopped projectors then +correctly received stream ingest rejection, retried against the bounded +pending queue and were eventually cgroup-OOM-killed. This is one shared +startup defect, not a new provider/data or topology issue. + +The approved completion packet therefore uses the already-built immutable +`qdl-v2-python:2.0.0-phase533-readiness-36914d2` +(`sha256:b75a414226d1990eea6b07022fdbd3d8cb9cad332294384f4ddb9010454e910e`) +for the existing V2 cache-user roles only: serially roll the already-passed +`query_v2_1`/`query_v2_2` if needed, then `stream_v2_active`, +`stream_v2_passive`, `projector_v2`, `projector_v2_2` and `projector_v2_3`. +No new role, worker, symbol process, topic, partition, identity or resource +limit is created. Each retains its command, 512 MiB cgroup, existing +`stable_state`, TLS and runtime mounts, consumer group and offsets. Normal +real canonical projection writes resume only after the active stream is +`READY`; there is no Kafka reset, Redis flush, SQLite deletion, V1, Rust, +ingestor, bar-edge, Trading System, alpha, broker or order-path change. + +**Runtime exit and rollback.** First require three mTLS `READY` samples from +each query reader; then roll stream active/passive one at a time and require +their cache, authority and lease components to be ready. Start each existing +projector sequentially with the same final image, requiring `running`, +restart `0`, `OOMKilled=false`, bounded Kafka/stream catch-up and a final +five-minute error scan without cache/gateway/OOM/fatal errors. Direct rollback +for any one role is its pre-packet image with unchanged mounts: reader +`sha256:7b5c848d2add9e0e36d88bfb837c93448140b7ade3a87d9e94e4902b2bf3f18e`, +stream `sha256:6859cfb0359e94816986e4e9b7a4a9f6486dd9fce6ac607b04e275086efbd42b`, +projector `sha256:36ed90b9a18e1c3bbd8cfc6169543f3a3ce1ce79f76f798e7777125cba9c63ed`. +Failure stops/recreates only the failed role and leaves all durable state +intact. Alpha no-order proof remains blocked until the seven-role V2 read and +projection plane passes. + +**Projector bootstrap correction (`PASS / PROJECTOR IMAGE NEXT`, 2026-09-01).** +One final read-only source inspection found the projector itself used +`spool.stats()` only to decide whether `RedisStableProjectionTarget.bind_cache` +should initialize an empty cache. It needs only the usage counter, not oldest, +newest or a full payload aggregate. It now uses the same bounded +`readiness_summary().records` as the reader health path. Query and stream roles +already running `sha256:b75a…e910e` do not need another recreate because their +runtime behavior is unchanged; only the three stopped projectors require the +successor immutable image from this source commit. + +**Evidence actually run:** host compile/diff checks passed; the isolated +non-root/read-only/no-network stable matrix reports **72 passed, 1 +intentionally skipped**. The added regression asserts that +`serve_stable_projector` uses the bounded usage summary and cannot regress to +`spool.stats()` at startup. Existing projector recovery, Kafka, stream +ingestion, catalog, cursor, L2, final-bar and five-symbol identity cases +remain in the matrix. + +**Revised final projector packet.** Build one immutable Python image from this +commit and update only `projector_v2`, `projector_v2_2`, and `projector_v2_3` +in the existing temporary override. Sequentially recreate those three stopped +roles. The active query/stream image remains the already-tested +`sha256:b75a…e910e`; its replacement is unnecessary and therefore excluded. +The prior projector image `sha256:36ed…c63ed` with the same mounts remains the +per-role rollback. No other component may be recreated. + +**Public SDK artifact correction (`IN PROGRESS / SOURCE-ONLY`, 2026-09-01).** +The first Python 3.10 alpha-runtime regression after the Phase 53.3 no-order +adapter exposed an artifact mismatch, not a data-plane defect: its vendored +`qdl-sdk==2.0.0` wheel has source digest +`1535f7f5cfb50050dc300a3b65471508ca9e10d4f3bcff0d9a9a9108cc23737e` and +predates the sealed V2 public fields `event_recency_policy`, +`max_session_liveness_ms`, and provider-session quality. The current public +SDK source has a different digest and the matching generated protobuf fields. +The existing 2.0.0 artifact must remain immutable; replacing bytes under the +same package version would break provenance and reproducibility. + +The approved narrow source scope is therefore one reproducible **`qdl-sdk +2.0.1`** patch artifact: update only the SDK version constants, SDK release +builder/CI assertions and version-specific SDK tests; build the wheel from +the already-sealed contracts; then let the alpha runtime pin the resulting +wheel by exact name/hash. Data Layer service version, Rust core, provider +adapters, V1/V2 API routes, manifests, runtime roles, Kafka, Redis, SQLite, +identities and consumer routing are excluded. The gates are a clean SDK +release build, exact Python 3.10 standalone import, current typed +`DataRequirement`/`QualityView` field validation, generated-contract presence +and the existing SDK release tests. The source rollback is to retain the +immutable 2.0.0 artifact and restore the alpha pin; no runtime rollback is +needed because this packet starts no Data Layer role. + +**SDK 2.0.1 source exit (`PASS / CONSUMER PIN NEXT`, 2026-09-01).** The +public SDK version and deterministic builder now produce +`qdl_sdk-2.0.1-py3-none-any.whl`; Data Layer service/OpenAPI remains `2.0.0`. +The builder produced SHA-256 +`2e6fbfa074876a14c9f43320d739067a3b980e4c4a86e309bdaefea7d3162529`, source +digest `2aac992269797848acc2177268f01f606dd69cba2239da1cb4c9110dd5c5b6c8` +and generated-contract digest +`2a25a601ae80b2ee4b36e6d1cb8ca44a13b8a037bf69a96f45659d1782a3a624`. +The artifact includes the current typed `DataRequirement` and `QualityView` +fields plus generated query protobuf, but no service internals. + +The isolated non-root/no-network/read-only Data Layer runner passed **20/20** +SDK release, feed-status, consumer-manifest and stable release tests. A +separate Python 3.10 alpha-runtime verifier imported the artifact and +constructed observed-event/session-liveness models as `2.0.1 OBSERVE LIVE`. +No Data Layer role, provider, Kafka, Redis, SQLite, manifest/authority or +endpoint was changed. The sole consumer action is the matching alpha-runtime +vendor pin; the prior immutable 2.0.0 wheel remains intact and rollbackable. + +**Phase 53.3 V2 read-plane runtime prerequisite (`PASS`, 2026-09-01).** The +bounded seven-role completion packet is now stable on the existing topology: +`query_v2_1` and `query_v2_2`, `stream_v2_active` and `stream_v2_passive` use +reader image `sha256:b75a414226d1990eea6b07022fdbd3d8cb9cad332294384f4ddb9010454e910e`; +`projector_v2`, `_2` and `_3` use +`sha256:288e617c7ec137dfcac5e94a5977772f2e0f39eac3ff48585980c0b26b0071bb`. +All seven were `running`, restart count `0`, `OOMKilled=false`; the primary +query healthcheck was `healthy`. A bounded ten-minute error scan found `0` +matches for `fatal|panic|oom|not_ready|unavailable|error` on every role. +Observed resident memory was bounded at roughly `110-126 MiB` for +query/stream and `326-347 MiB` for the three live projectors within their +existing `512 MiB` cgroups. + +This completion used no new topology, topic, offset reset, Redis flush, +SQLite deletion, V1/Rust/ingestor/bar-edge change, Trading System, alpha, +broker or order-path mutation. It establishes only the V2 query/stream/cache +prerequisite for the next isolated alpha no-order proof; it is not itself +consumer acceptance or authority promotion. + +### Phase 53.3 - Alpha Entitlement And Advisory-Context Convergence + +**Status:** `IN PROGRESS / SOURCE-ONLY` (2026-09-01). + +**Goal and approved scope.** Complete the source prerequisite for the +representative alpha no-order proof without widening execution authority. The +two stable alpha consumer manifests must derive their Binance USD-M/OKX Swap +five-liquid requirements from the existing canonical demand/catalog, including +native final BAR intervals, TRADE, QUOTE, depth-100 L2 snapshot/delta and +provider-supported reference data. Alpha remains `ALPHA` grade with +`execution_dependency=FORBIDDEN`; only the Trading System Risk consumer owns +execution-grade cache admission. A manifest requirement is an entitlement, +not a subscription: shared Rust ingestors/core/bar-edge remain the sole data +plane and no symbol-specific worker, image, topic, identity or container may +be introduced. + +**Invariants and decision boundary.** The compiler must be deterministic and +idempotent, derive native identity/policy/freshness from reviewed source +artifacts, retain exact V1 fallback only where the release route already +proves it, and make quote/mark/L2/reference routes `BLOCKED` rather than +silently falling back. Existing `1..100` manifest capacity is too small for a +least-privilege five-liquid alpha (Binance needs 125 routes and OKX 110), while +the existing shared Trading-System source demand already declares 186 routes. +The loader boundary may therefore become `1..256`; it is a declaration limit, +not a request, stream or warmup concurrency increase. Per-request quotas stay +bounded (`max_batch_items <= 100`, `max_warmup_rows <= 10,000`). DNSE/VN stays +unchanged and V1-primary. No runtime bundle, image, role, provider request, +Kafka/Redis/SQLite data, Trading System/alpha process, order or broker action +is allowed in this source slice. + +**Required source gates.** Compile and contract tests must prove both rendered +manifests load; exactly five native instruments per venue; 70 native BAR, +5 TRADE, 5 QUOTE, 5 BOOK_SNAPSHOT and 5 BOOK_DELTA routes per venue; reference +coverage only where the canonical reference manifest supports that venue; +`ALPHA` grade/no execution privilege; venue/symbol isolation; exact policy and +fallback decisions; quota bounds; deterministic re-render; and release/primary +route integrity. The later real packet renders a sealed binding SHA from these +artifacts and runs the named `--rm` no-order alpha proof. Source revert is +rollback. + +**Source/config exit (`PASS / SEALED RUNTIME PACKET NEXT`, 2026-09-01).** A +deterministic compiler now materializes the two alpha manifests from the +approved stable crypto demand, source catalog and reference/L2 manifest; it +does not enumerate symbols, provider calls or subscriptions itself. The +rendered `alpha.binance.paper.stable` revision `9` has `125` requirements and +`alpha.okx.paper.stable` revision `8` has `110`: each venue has exactly five +native perpetual instruments, `70` final BAR routes, and five each of TRADE, +QUOTE, BOOK_SNAPSHOT and BOOK_DELTA, plus only reference products supported by +the existing canonical reference entitlement. The shared manifest declaration +limit is now `1..256`, while request/batch/warmup/stream quotas stay bounded; +this accommodates the `125`/`110` declared entitlements and the pre-existing +`186`-requirement Trading System manifest without increasing runtime +concurrency. + +The release route was deterministically regenerated to revision `15` and the +primary-consumer route to revision `4`. It now contains `303` products: +`5` monitoring, `61` Trading System, `125` Binance alpha, `110` OKX alpha and +`2` unchanged VN products. V1 fallback is retained only for admitted Binance +TRADE routes; final BAR, quote, L2 and reference products fail closed rather +than silently changing source. The V2 paper acceptance projection has `299` +V2-primary products (`234` durable, `65` on-demand); its five-liquid +Trading-System/Binance-alpha/OKX-alpha subset has `295` (`230` durable, `65` +on-demand). Alpha quote context is deliberately `ALPHA` grade with `5s` +freshness, while the Trading System/Risk execution-grade quote remains `2s`; +Risk independently re-reads the latter before normal or preview admission. + +Evidence: the isolated non-root, read-only, no-network Data Layer runner +passed **32/32** entitlement/release/fallback/consumer tests, **9/9** native +final-BAR materialization tests and **54/54** reference/L2/contract/SDK +consumer tests. The compiler dry run returned no changed files after apply; +`compileall` and `git diff --check` passed. These are source/config gates +only: no runtime bundle/image/role, provider request, Kafka/Redis/SQLite +state, Trading System, alpha process, execution session, paper order or broker +action was created. The next and only remaining Phase 53.3 runtime decision is +the separately scoped sealed-binding, temporary `--rm` alpha/Gateway no-order +packet with pre/post mutation evidence. Source rollback is this commit's +revert; V1/VN remain unchanged. + +## Cross-Repository V2 Alpha Consumer Production Rollout + +**Status:** `APPROVED / PHASE A IN PROGRESS` (2026-09-01). The detailed +cross-repository contract is recorded in Trading System plan Section 54. This +Data Layer entry owns the binding compiler, immutable reader release and +real-provider acceptance evidence. + +### Data Layer Responsibilities + +1. **Phase A.** Merge the current V2 source branch to `dev` only after source + gates pass there. Record the Data Layer SHA and public `qdl_sdk` version in + the release tuple; keep `main` unchanged until stable certification. +2. **Phase B.** Replace the five-liquid-only declaration with a deterministic + config-derived binding compiler. It must consume alpha requirements for + native Binance USD-M/OKX Swap identity, final BAR/history, `maxlen` and + `min_bars`, TRADE/QUOTE/MARK/BOOK, reference products and exact fallback/ + freshness policy. It emits a sealed, versioned, secret-free binding per + alpha deployment or a typed `BLOCKED` reason. It does not create a + subscription, symbol worker, image, topic or identity per symbol. +3. **Phase C.** Build one canonical immutable Python reader/query/stream image + from the merged `dev` SHA and seal its runtime bundle. Replace only reader + roles with per-role rollback digest. V1, Kafka, Redis, SQLite, Rust core, + ingestors, bar-edge and projectors stay intact unless a source change makes + their replacement necessary. +4. **Phase D.** Supply real provider V2 data for paired Binance/OKX no-order + alpha proof. A successful route requires native identity, finality, + authoritative/complete/no-gap quality, cursor replay/reconnect and bounded + resource evidence. A product that needs unavailable data must be `BLOCKED`, + never represented as zero/synthetic/fallback data. +5. **Phase E.** Publish only a release-tagged image/config after paper-canary + evidence. Retain V1 as named rollback. After release, remove only confirmed + unreferenced test images/cache and merged worktrees, recording disk + before/after and retention coordinates. + +### Data Layer Acceptance Rules + +- Binance and OKX are paired test venues for each eligible alpha class; source + and provider data remain native to their venue. +- Support declared warmup windows through `10,000` rows while preserving + bounded request/batch/stream quotas and final-BAR-only execution inputs. +- Reference data and L2 become execution inputs only when an exact consumer + requirement/policy permits them. Alpha remains `ALPHA` grade; Trading System + Risk retains the independent execution-grade reread. +- DNSE/VN remains V1-primary and outside this crypto rollout. + +**Phase A source-gate journal (`PASS / DEV INTEGRATION NEXT`, 2026-09-01).** +The V2 binding/compiler branch was revalidated in an isolated, network-disabled +and read-only container before `dev` integration. `139` relevant source tests +passed with `1` documented skip across alpha entitlement compilation, +query-readiness, five-liquid/native final-BAR materialization, stable reader +recovery, reference/L2 materialization and SDK stream/feed contracts. The +SDK release build also reproduced `qdl-sdk==2.0.1` with wheel SHA-256 +`2e6fbfa074876a14c9f43320d739067a3b980e4c4a86e309bdaefea7d3162529`. +`compileall` and `git diff --check` passed. Recovery/overlap log records came +from deterministic negative test cases and no provider was contacted. + +No V2 role, bundle, image, Kafka/Redis/SQLite state, V1 service, Trading +System, alpha process or order path changed. The next valid Phase A operation +is merge to `dev`; runtime image/build and reader rolling remain Phase C only. + +**Phase A integration closure (`COMPLETE / DEV INTEGRATED`, 2026-09-01).** +The tested source payload is now contained by remote `dev` at +`dc4ca8ed534de516328f63b2904d0d7d248f3969`; `main` is unchanged. The +cross-repository source tuple for the V2 rollout is: + +```text +data_layer payload SHA: dc4ca8e +trading_system payload SHA: dcd6cd1 +execution_alpha payload SHA: 76b110b +qdl_sdk wheel: 2.0.1 +``` + +This records source convergence only. It does not certify any existing reader +runtime as the new binding compiler, nor authorize a service roll. Phase B +starts from this `dev` baseline; Phase C is still the first runtime mutation. + +### Phase B Config-Derived Binding Compiler (`IN PROGRESS / SOURCE-ONLY`, 2026-09-01) + +**Approved scope.** Replace the five-liquid-only compiler input with a portable +deployment-requirement inventory exported from `execution_alpha` configuration +and reconciled against the corresponding Compose service environment. Each +eligible Binance USD-M or OKX Swap deployment must declare its native identity, +final BAR interval, warmup/maxlen/min-bars, typed realtime/reference/L2 feeds, +freshness/finality/gap policy and fallback. The Data Layer compiler resolves +only catalog/capability-backed slices and emits a secret-free, deterministic +sealed binding or an explicit `BLOCKED` result. It creates no subscription, +provider call, symbol worker, topic, identity, image, bundle or runtime state. + +**Invariants.** The compiler is provider-neutral after native identity +resolution; Binance and OKX requirements stay venue-native and are both +rendered for every representative alpha class. Existing V1 routes and the +five-liquid manifests remain compatibility inputs until the later Phase C +bundle replacement. DNSE/VN remains V1-primary. `maxlen` may be 1..10,000; +missing/optional metrics never become zero. A config/Compose mismatch, unknown +catalog identity, unsupported feed or missing required capability fails closed +with a typed result rather than a partial binding. + +**Required source gates.** Tests must cover deterministic re-render; all active +Binance/OKX Compose deployments being represented as an admitted binding or +typed block; 2,500/5,000/10,000 history bounds; multi-symbol, bracket, grid/L2 +and basis/reference profiles; Binance/OKX identity isolation; capability +failure; and portable Trading System binding parsing. Source-only test runs use +network-disabled/read-only containers where dependencies require Docker. No +runtime service, provider, Kafka, Redis, SQLite, V1, Trading System, alpha, +execution session, order, broker state or deployment configuration may change. + +**Rollback and decision boundary.** This phase is reverted solely by reverting +its source commits on `dev`. A new immutable reader image and any rolling change +remain Phase C decisions and are explicitly excluded here. + +**Compiler and portable-SDK slice (`PASS / SOURCE-ONLY`, 2026-09-01).** Added +`scripts/compile_alpha_deployment_bindings.py`, a deterministic compiler which +consumes the portable Execution Alpha inventory and emits only the existing +`qdl.v2.consumer-route-binding.v1` contract. It resolves each required route +through the stable source catalog and reference/L2 entitlement, preserves +native Binance USD-M and OKX Swap identity, retains no alpha execution-grade +privilege, and emits one typed `BLOCKED` result rather than a partial binding +when any required route is unavailable. The output writer is explicit and +caller-owned; Phase B created no mounted binding or release artifact. + +The real Compose inventory was exported in a disposable, network-disabled, +read-only alpha container: `93` records with SHA-256 +`e77a93363b59561a95053faa02afb2634d2d6e0b82e0e0366bcfdc943dbbcda4`. +The compiler admitted `17` exact bindings containing `106` V2 products and +blocked `76` records. The blocked set is fully explained: `35` non-paper +deployments, `32` unreviewed profiles, `8` DNSE/VN V1-primary deployments, and +one native OKX basis probe whose `*-USDT-SWAP` route has no declared +same-family basis capability. No unavailable metric was converted to zero or +silently sourced from another venue. A Trading System SDK-only container parsed +all `17` bindings and selected all `106` exact routes; all were advisory +(`execution_grade=false`), and every OKX no-order probe had zero V1 fallback. + +The isolated no-network/read-only Data Layer regression command ran `22/22` +tests across the new compiler plus current entitlement/reference/universal +release contracts. It also exposed and corrected a real shared-contract gap: +metric series (`OPEN_INTEREST`, `LONG_SHORT_RATIO`, `TAKER_FLOW`, `BASIS`) may +carry their explicit sampling interval, while point-in-time products remain +interval-free and fail closed. No Data Layer service/image/provider/Kafka/Redis +SQLite/V1/Trading System/alpha/order state changed. Source rollback remains a +revert; Phase B is still open only for cross-repository commit/`dev` integration. + +**Optional-metric and cross-repository source exit (`PASS / DEV INTEGRATION NEXT`, +2026-09-01).** The compiler now distinguishes an unavailable optional metric +from a required route failure. A capability-backed optional route is excluded +from the sealed binding and recorded in `optional_unavailable` with exact +venue/native identity/feed/interval/policy/reason; it is never represented as +zero, silently re-routed, or permitted to be the only admitted route. Unknown +identity, malformed booleans and unavailable required routes still fail +closed. The isolated Data Layer regression command now passed `23/23` tests, +including optional-unavailable, no-admitted-required-route, checksum, +determinism, reference/L2 and universal-release cases; isolated Ruff passed. + +A fresh real-Compose source proof exported `93` records with inventory SHA +`e77a93363b59561a95053faa02afb2634d2d6e0b82e0e0366bcfdc943dbbcda4`. +Two identical compiler writes produced the same compilation SHA +`940b2fe002b0c14d5da0181352c20fe1d6ae163fc6d10bde55c90c48b33a0ba4`; +the second changed no files. It admitted `17` bindings/`106` routes and +typed-blocked `76` records (`35` out-of-paper, `32` profile review, +`8` DNSE/V1-primary, `1` unavailable native OKX basis). A separately +isolated Trading System parser proof accepted every binding, preserved four +`1d` metric-series routes, found zero execution-grade alpha routes and zero +OKX V1 fallbacks. These are source-only artifacts in a scoped temporary +directory: no provider, image, runtime role, Kafka, Redis, SQLite, V1, +Trading System, alpha, execution or broker state changed. Phase B is ready +only for its three coherent commits and fast-forward integration into `dev`; +Phase C remains the first permitted runtime handoff. + +**Phase B integration closure (`COMPLETE / DEV INTEGRATED`, 2026-09-01).** +The tested compiler source is now in remote `data_layer:dev` at +`ee002b137d6cc0c7918a30dc52f30cbb28d74b78`; `main` and all runtime roles +remain unchanged. The next permitted action is Phase C's separately recorded +canonical reader-image/bundle packet. It must seal this inventory/binding +generation before any reader roll and retain V1 plus per-role rollback digests. + +**Phase B scoped cleanup (PASS, 2026-09-01).** Deleted only the two temporary +cross-repository proof directories under /tmp (qdl-phaseb-bindings.*, 852 KiB +total) and two unreferenced test-only images: +execution-alpha-runtime-numba:0.1.1 (sha256:7f1df0...dc5e9, 369 MB) and +tradingsystem-test:latest (sha256:f10829...de08, 192 MB). No test container +remained because every test used --rm. Disk changed from 203/290 GiB used +(88 GiB free) to 201/290 GiB used (90 GiB free). The active reader image +qdl-v2-python:2.0.0-phase533-readiness-36914d2 (sha256:b75a41...10e) and both +query/stream replicas remained running; no runtime restart occurred. No broad +BuildKit prune was performed because its shared-cache retention set was not +proven. + +### Phase C - Canonical Reader Release And Sealed Alpha Binding Bundle + +**Status:** `COMPLETE / DEV INTEGRATED` (2026-09-01). + +**Detailed procedure:** `docs/runbooks/phase54-config-derived-reader-release.md`. + +**Goal.** Replace the phase-named reader artifact only with one canonical, +immutable Data Layer image built from the merged `dev` source, while preparing +the exact config-derived alpha binding bundle that later no-order alpha pairs +will mount read-only. This phase does not create an alpha-specific service, +symbol worker, topic, identity or image. + +**Source and artifact coordinates.** The source baseline is `data_layer:dev` +`6967573552733cdaf8576cd8ef95ef95960f793f`. The final candidate tag must be +`qdl-v2-python:2.0.0-dev-` and its OCI revision must equal +that exact source SHA; a mutable tag is never the runtime pin. The first +preparer gate built `qdl-v2-python:2.0.0-dev-8db07ea` from +`8db07ea9f7d3ae27c31575c9a14fb7e6b54376be`, inspected image +`sha256:376d474759c96bab1b3ac75a0c9311097fe3c93ba310b427f97afa963f58f77a`, +and passed its isolated selected matrix (`23` tests). It is a build-gate +artifact only: the final image is rebuilt after the source-managed bundle +preparer lands and receives its own exact SHA/tag. The secret-free release +bundle lives outside every checkout under a newly named +`/home/bobby/.local/state/qdl-v2/releases/` directory and contains only the +verified inventory, its `17` sealed bindings, compilation report, source/image +digests and candidate/rollback four-role image overrides. It does not contain credentials, +private keys, provider payloads, cursors, SQLite, Redis/Kafka data or logs. + +**Approved runtime scope.** After candidate source/image gates pass, serially +recreate only `query_v2_1`, `query_v2_2`, `stream_v2_active` and +`stream_v2_passive` in Compose project `qdl_v2_stable_candidate`. The active +reader image is `qdl-v2-python:2.0.0-phase533-readiness-36914d2` +(`sha256:b75a414226d1990eea6b07022fdbd3d8cb9cad332294384f4ddb9010454e910e`) +and is the exact four-role rollback image. V1, Kafka topology/offsets, Redis, +SQLite, Rust core, both ingestors, bar edge, projectors, Trading System, alpha +containers, broker credentials and order path are excluded. Normal reader +startup reads are allowed; no new provider subscription or durable-store reset +is part of this packet. + +**Runtime hygiene invariant.** Docker labels showed the serving readers were +originally launched from a removed phase worktree and a stack of historical +overrides. The successor invocation must use the canonical checkout +`/home/bobby/data_layer` and one new release-owned override as the final image +layer; it may retain only the existing required base/C2 selectors that are +proven in rendered config. It must not rely on a `/tmp` override, a feature +worktree or recreate an unrelated role merely to normalize labels. The +canonical root's base/C2 Compose files are byte-identical to this Phase C +worktree (`60624...c144`, `8013...fca3`), so a same-image serial label +normalization of the four approved roles is permitted after their first image +handoff; it does not alter the rendered service specification or data plane. + +**Build/test gates.** Build exactly one image. Inspect its OCI revision, +non-root user and digest; run the selected binding/release/reference/L2/query/ +stream unit matrix inside that immutable image with network disabled, read-only +filesystem and tmpfs-only test state; run `docker compose config --quiet` with +the sealed public selector files; compile/re-render the binding bundle twice +and verify all hashes. Before every serial reader recreate record image, +restart count, health, manifest checksum, lag and RSS; after each, verify the +same and stop/rollback the just-changed reader if it is not healthy. The later +Phase D no-order alpha proof, not this phase, is the first permitted alpha or +Gateway/Risk mutation. + +**Stream readiness semantics.** `stream_v2_active` and +`stream_v2_passive` are cooperative lease replicas, not two independently +ready writers. Their names do not assert ownership. At any sampled point the +acceptance condition is exactly one replica with `/health/ready` `200` and a +`READY` `gateway_lease`, while the other returns `/health/dependencies` `200` +with every non-lease component `READY` and `gateway_lease=STANDBY` (its +`/health/ready` is intentionally `503`). Both must remain live, restart-free +and on the candidate image. Zero leaders, two leaders, an unexpected component +state or a lease-error is a rollout failure. Pre-roll inspection observed the +expected shape: query replicas and named active stream ready; named passive +stream standby with consumer manifests/cache/Redis/authority all ready. +For every serial stream handoff, recreate the currently observed `STANDBY` +replica first and the current `READY` lease holder second; a fixed container +name order would be wrong after a prior healthy handoff reverses ownership. + +**Increment 1 - sealed bundle preparer (`IN PROGRESS / SOURCE-ONLY`, +2026-09-01).** The bundle is a durable release coordinate rather than an +ad-hoc shell copy. Add one small source-managed preparer and deterministic +unit test. It may only validate the already sealed inventory/compilation, +write a new caller-owned output directory atomically, render the four fixed +reader image service overrides and record hashes/provenance. It must reject an +existing output directory, unknown/duplicated reader roles, malformed image +digests, a report/inventory digest mismatch, or any attempt to write outside +the caller-owned output. It must not parse, copy or emit a runtime env file, +certificate, identity, cursor, provider payload, Kafka/Redis/SQLite state or +secret. The inventory export itself is read from an ephemeral `origin/dev` +archive of Execution Alpha, never from the detached user checkout or a +long-lived duplicate worktree. The archive is removed once the sealed bundle +hash has been verified twice. + +**Increment 1 implementation (`PASS / SOURCE-ONLY`, 2026-09-01).** Added +`scripts/prepare_alpha_reader_release.py` and +`tests/test_alpha_reader_release.py`. The preparer accepts only an inventory, +the compiler's named binding directory/report and explicit immutable release +coordinates. It verifies inventory/report/binding checksums and canonical SDK +bindings, creates an output directory once through a private staging rename, +and writes only `inventory.json`, `bindings/`, `compilation-report.json`, +`reader-image.override.yml` and `release-manifest.json`. It rejects malformed +coordinates, a candidate equal to rollback, non-canonical/tampered bindings, +input/report mismatch and an existing/unsafe output path. The override has the +fixed four reader services only. No env, TLS, identity, runtime state, provider +payload or secret is accepted or emitted. Isolated source-mounted, no-network +tests passed: `python -m unittest -q tests.test_alpha_reader_release +tests.test_alpha_deployment_bindings` (`10` tests), followed by +`python -m compileall -q scripts/prepare_alpha_reader_release.py +tests/test_alpha_reader_release.py` with bytecode redirected to tmpfs. Both +commands used the non-root candidate image with read-only source; runtime +mutations and order actions remained `0`. The final candidate image must be +rebuilt from the commit containing this preparer before the private release +bundle is materialized. + +**Increment 1 rollback-selector correction (`IN PROGRESS / SOURCE-ONLY`, +2026-09-01).** The initial preparer records the rollback image ID but not its +companion Compose selector. Before any service roll it must accept and validate +the explicit rollback image reference, write a fixed four-role +`reader-rollback.override.yml`, and bind both reference/ID pairs into the +manifest. This is an in-scope correctness correction, not a new runtime +feature: rollback must never depend on reconstructing a command manually or on +a mutable historical tag. + +**Rollback and close boundary.** Source rollback is a revert. Runtime rollback +is a serial four-role recreate using the exact retained `b75a...e910e` image +and pre-roll compose selector set. This phase closes only after the image is +immutable, the secret-free bundle is sealed, four-reader rollout health passes +and the obsolete phase-named active worktree/temporary override is no longer a +runtime dependency. It does not certify an alpha consumer or paper order. + +**Phase C close (`PASS / RUNTIME HANDOFF COMPLETE`, 2026-09-01).** Source +commits `95f84cd` and `d619be6` added the sealed bundle preparer and explicit +rollback selector. The final immutable reader image is +`qdl-v2-python:2.0.0-dev-d619be6` +(`sha256:e5cea2afa405188293e28fa8b1fd1a6ac22b2b62db8aa2e95efc44913b407963`), +with OCI revision `d619be6f64f0ec00144d39773db70e9c1f2ba6c0`, non-root +`qdl:qdl`, and `27/27` scoped unit/contract tests passed inside that image +with no network, read-only filesystem and tmpfs bytecode state. The real +`execution_alpha:origin/dev` archive was exported twice in isolated no-network +containers: both inventories were `93` deployments with SHA +`e77a93363b59561a95053faa02afb2634d2d6e0b82e0e0366bcfdc943dbbcda4`. +Both compilations produced `17` admitted and `76` typed-blocked deployments, +`106` products and compilation SHA +`940b2fe002b0c14d5da0181352c20fe1d6ae163fc6d10bde55c90c48b33a0ba4`. +The two sealed bundles were byte-identical; final release bundle +`/home/bobby/.local/state/qdl-v2/releases/2.0.0-dev-d619be6` has manifest SHA +`dba055812ab87eb836895fd71c3dfd5573b2c9ba7f4c589be2c956881e1756c5` and +contains candidate and exact rollback selector for only the four reader roles. + +**Bounded runtime handoff (`PASS`).** The first serial candidate handoff +verified the four services, then the final canonical-root normalization +recreated exactly `query_v2_2`, `query_v2_1`, the observed stream standby and +the observed stream leader using only +`/home/bobby/data_layer/docker-compose.v2-stable.yml`, its byte-identical C2 +override (`8013...fca3`) and the sealed candidate override. V1, Kafka, +Redis, SQLite, Rust, ingestors, bar edge, projectors, Trading System, alpha, +database and order/broker paths were not recreated or reset. Every final role +uses `e5cea...7963`, is running with restart count `0` and `OOMKilled=false`; +both query mTLS readiness probes returned `200`; stream active reported exactly +one `READY` lease and stream passive exactly one clean `STANDBY`, with all +other dependency components ready. Runtime labels now point to canonical +`/home/bobby/data_layer` and the sealed bundle, not `/tmp` or any feature +worktree. Reader/stream lag has no active alpha consumer in this phase and is +therefore intentionally not asserted; Phase D owns real cursor/reconnect/lag +evidence with paired Binance/OKX no-order consumers. + +**Scoped cleanup (`PASS`).** Removed exactly the two unreferenced preparer +images `376d...77a` and `70c8...d49`, the superseded +`2.0.0-dev-95f84cd` bundle and `/tmp/qdl-phasec-execution-alpha.0P7bHe`. +Retained only the active reader image `e5cea...7963` and named reader rollback +image `b75a...e910e` for this packet. No broad BuildKit prune, volume/network, +source or runtime-state deletion occurred. Disk moved from `204/290 GiB` used +(`86 GiB` free) to `203/290 GiB` used (`87 GiB` free), and post-cleanup mTLS +readiness/lease checks still passed. The next permitted action is Phase D's +four representative paired Binance/OKX no-order proof; no alpha or order was +started by Phase C. + +### Phase 54 Execution Ledger - Approved Five-Phase Consumer Rollout + +**Status:** `APPROVED / PHASE A ACTIVE` (2026-09-01). + +This ledger makes the cross-repository rollout executable without changing its +established contracts. It is governed by this section, Trading System Unified +Plan Section 54, and the Alpha Runtime Migration Architecture rollout entry. +Every source slice integrates through `dev`; `main` remains release-only until +Phase E certification. + +**Shared invariants.** Each eligible crypto acceptance is a paired native +Binance USD-M and OKX Swap observation. V1/V2 parity is compared only within +the same venue/candle corpus; no test asserts Binance-to-OKX strategy parity. +DNSE/VN remains V1-primary. Provider data is real or durable provider-byte +replay. No alpha, symbol or interval creates a dedicated Data Layer worker, +image, topic or identity. + +| Phase | Scope | Required exit evidence | Runtime boundary | +| --- | --- | --- | --- | +| A | Converge tested V2 source in all three `dev` branches and record one release tuple. | ancestry/range-diff, source suites on `dev`, SHA tuple and `qdl_sdk` version. | source only | +| B | Compile sealed binding from each real alpha Compose/config declaration. | every eligible crypto deployment admitted or typed `BLOCKED`; 2,500/5,000/10,000 history bounds; native identity. | source only | +| C | Build canonical reader image/bundle and roll shared V2 readers. | SHA -> digest -> bundle -> role/rollback map; healthy query/lease pair. | four reader roles only | +| D | Prove four representative alpha classes in concurrent Binance/OKX no-order pairs. | real V2 warmup/cursor/reconnect/final BAR, declared L2/reference, Risk reread and exact zero mutation. | disposable consumers only | +| E | Run admitted paper canaries and close release. | same-venue input parity, observable V1 fallback, paper lifecycle/capacity, cleanup inventory. | declared paper scopes only | + +**Phase closure discipline.** Each coherent slice is tested, journaled and +committed with the configured user identity, then merged to `dev`. Only after +ancestry/reference verification may its feature worktree/branch and exact +test-only artifacts be removed. A reviewed release PR is the only route from +`dev` to `main`. + +#### Phase B Full Config Inventory Completion (`IN PROGRESS / SOURCE-ONLY`) + +**Goal.** Complete the already-landed compiler so every real eligible crypto +deployment from Execution Alpha is represented by a declarative requirement +record and becomes either an exact sealed V2 binding or a typed actionable +`BLOCKED` result. The historical five-liquid manifest remains compatibility +input only; it is never the default source of alpha truth. + +**Approved scope.** Requirement records carry native `venue`, `product` and +instrument identity; final signal BAR interval; warmup `maxlen`/`min_bars`; +TRADE/QUOTE/MARK_INDEX/BOOK needs; reference series; freshness/finality/gap +policy; required/optional status; and allowed fallback policy. Values must be +derived from the deployment Compose/config boundary, not strategy code. Each +crypto family gets a concurrent native Binance USD-M and OKX Swap test route; +a Binance-only economic model receives an explicit native OKX data probe, not +a fabricated equivalent strategy. DNSE/VN remains V1-primary. + +**Invariants.** `maxlen` accepts only `1..10,000`; non-final history cannot +become execution state; identity is exact and venue-native; optional absence is +recorded rather than zero-filled; required absence is fail-closed. The compiler +does not create provider demand, a worker, topic, image, identity, runtime +bundle or persistent state. + +**Required tests and exit.** Cover every rendered Compose service, all declared +Binance/OKX pairs, 2,500/5,000/10,000 warmup bounds, single/multi-symbol, +portfolio, bracket, grid/L2 and basis/reference classes, malformed config, +unknown instrument, missing required capability, optional omission, duplicate +identity and deterministic double render. Data Layer parser plus Trading System +portable parser must select the same route set. Phase exits only when the full +inventory has a hash, each record is `ADMITTED` or typed `BLOCKED`, and no +secret/runtime mutation occurred. Revert only the Phase B source commits on +`dev` if required. + +**Phase A revalidation (`PASS / DEV FAST-FORWARD NEXT`, 2026-09-01).** The +requested V2 source refs are already ancestors of `origin/dev`: Data Layer +`origin/fix/phase533-v2-query-readiness` at `5cf8e21`, Trading System +`origin/feat/alpha-sdk-v2-convergence` at `ce8daa9`, and Execution Alpha +`origin/feat/alpha-sdk-v2-convergence` at `f78c40d`. In all three repositories +`origin/main` is also an ancestor of `origin/dev`; no history rewrite or +blind merge is needed. The Data Layer feature branch, based on `dev`, passed +the isolated read-only/no-network binding/release/entitlement/reference/L2 +suite: **27/27**. No provider, runtime role, Kafka/Redis/SQLite, Trading +System, alpha, order or broker state was touched. Phase A next records the +three committed integration SHAs and fast-forwards `dev` only. + +**Phase A integration closure (`PASS / DEV PUSH PENDING`, 2026-09-01).** The +three source lines now converge without a history rewrite: Data Layer +`963939d` (`dev` base), Trading System `613ef15`, and Execution Alpha +`a8cb775`. Both consumer commits were fast-forwarded from their respective +official local `dev` branches after their source-only gates passed. The shared +SDK coordinate is the reproducible Data Layer `qdl-sdk==2.0.1` artifact +`2e6fbfa074876a14c9f43320d739067a3b980e4c4a86e309bdaefea7d3162529`; its +wheel, SBOM and manifest were compared byte-for-byte before acceptance. + +No reader, stream, alpha, provider, Kafka/Redis/SQLite, PostgreSQL, command, +order, session, position or broker state was changed. The remaining action in +this source slice is to commit this ledger update, push the three `dev` +branches, verify ancestry, and then begin Phase B's real-config compiler proof. + +**Phase A cross-repository closure and cleanup (`COMPLETE`, 2026-09-01).** +The source-convergence commits pushed to `origin/dev` are Data Layer +`e235f84`, Trading System `613ef15` and Execution Alpha `a8cb775`; this +follow-up journal records their verified cleanup. All required source ancestry +checks passed; `main` remains unchanged. No Data +Layer image or runtime role was created by this source phase. The two +unreferenced cross-repository test images were removed only after a +zero-container-reference check; the image set fell from `67.87 GiB` to +`66.15 GiB` and host disk from `203/290 GiB` to `202/290 GiB`. Existing +BuildKit cache, V1/V2 runtime images, volumes, topics, databases and running +services were retained. Phase B is the sole active work item. + +**Worktree closure (`COMPLETE`, 2026-09-01).** The clean +`feat/v2-alpha-binding-completion` worktree and merged local branch were +removed after remote-`dev` ancestry verification. The canonical `main` +checkout and the separately retained `fix/phase533-python310-sdk` worktree +were not changed. No source or runtime artifact was deleted outside this +closed source phase. + +**Phase B current inventory/compiler proof (`PASS / SOURCE-ONLY`, 2026-09-01).** +The current Execution Alpha Compose/config export (inventory SHA-256 +`738363ab1c8a56f75a6e1b728d96dab59b7632c09023d6a28a7e3b72068e0d60`) was +compiled twice inside the existing `qdl-v2-python:2.0.0-dev-d619be6` image +with network disabled, source read-only and caller-owned `/tmp` outputs. Both +renders were byte-identical: compilation SHA-256 +`1a859f933086715fca40e5efc1bd8293a9cc25a86b03a1bab3b6f224557cf2c6`, +`ADMITTED=17`, `BLOCKED=76`, and `106` exact V2 products. Each output is a +derived disposable artifact only; it created no provider demand, image, +identity, runtime bundle, Kafka/Redis/SQLite state, alpha, session, order or +broker action. + +The one blocked paired probe is intentional and explicit: +`basis_arb_binance:...:okx-probe` needs native OKX `BASIS` under +`crypto_liquid_v2`, which the current capability catalog has not certified. +It is therefore `CAPABILITY_UNAVAILABLE:BASIS:crypto_liquid_v2`, rather than +being supplied by Binance or a fabricated equivalent. All other required +routes are sealed native mappings. The downstream Trading System parser +interoperability correction is recorded in its Phase B journal; Phase B does +not close until that source commit is integrated to `dev`. + +**Phase B integration candidate (`IMPLEMENTED / CI PENDING`, 2026-09-01).** +The compiler evidence journal is now on remote `data_layer:dev` at `4b49efe`. +The three caller-owned inventory/compiler directories (724 KiB total) were +deleted after byte-identity and parser proof; no image was built and no Docker +cache was created by this phase. Docker image storage remains `66.15 GiB`; +the active/referenced image set and all V1/V2 runtime state were intentionally +retained. `main` remains unchanged. The only Phase B source correction lies in +Trading System route parsing and is awaiting its own CI result; then the +canonical shared-reader release packet of Phase C is the next allowed action. + +**Phase B CI regression correction (`IN PROGRESS / SOURCE-ONLY`, 2026-09-01).** +The baseline `unit-tests` failure was reproduced in a read-only/no-network +container at `test_catalog_demand_consistency`. Its negative pass-through +fixture selected the first `FRESH_SNAPSHOT` requirement without constraining +the feed; the expanded reference manifest now orders a `BASIS` requirement +first. `BASIS` correctly remains ineligible for provider BAR pass-through, so +the test fixture rather than the policy is stale. The registered alpha BAR +requirements correctly require `SNAPSHOT_AND_REPLAY`; the fixture must derive +one `FRESH_SNAPSHOT` variant from a declared BAR before testing pass-through. +The same targeted run exposed a second stale assertion: it assumed five +Binance `15m` and five OKX `1h` BAR requirements, while the expanded paired +manifest declares both `15m` and `1h` for each of five native instruments. +The repair must compare the full paired 20-route set independently of YAML +ordering. It must retain the replay and unknown-instrument negative +assertions, and must not change provider admission, fallback, catalog, demand, +image, runtime or any data plane. + +**Phase B full-suite fixture reconciliation (`IN PROGRESS / SOURCE-ONLY`, +2026-09-01).** The complete CI-equivalent suite ran `1,272` tests in a +network-disabled, read-only container with an ephemeral `/app/logs` mount and +found `3` stale count assertions plus `3` selectors that assumed the prior +five-liquid/pass-through manifest. The source-of-truth manifests now declare +`110` Phase-10.3 products, all durable; `76` exclusions; `303` release routes +(`299` V2 and `4` V1); and a legal alpha warmup maximum of `10,000` rows. The +pass-through semantic checks remain necessary but are now unit-derived from a +declared `15m` BAR with only its recovery policy changed to `FRESH_SNAPSHOT`. +The narrow repair updates test expectations and that synthetic test fixture +only. It does not alter the manifest, catalog, provider admission, fallback, +durability, maxlen policy, image, runtime or data plane. + +**Phase B durable-history bound repair (`IN PROGRESS / SOURCE-ONLY`, +2026-09-01).** Re-running the affected suite proved one real configuration +defect instead of another stale assertion: the consumer contract permits +`10,000` final BAR warmup rows, but the stable BAR edge Compose value and its +constructor capped warmup/catch-up at `1,000`; the Binance REST edge also made +only one `<=1,000`-row request. The repair is deliberately shared and +provider-neutral at the core boundary: the Python Binance vendor adapter gains +bounded backward pagination with per-page retry, end-boundary, duplicate, +continuity and no-progress checks; the stable edge retains one global bound but +permits `1..10,000` for both bootstrap and gap repair; the existing OKX +adapter's bounded 10,000-page implementation remains unchanged. Compose will +declare both bounds as `10,000`, matching the approved public V2 maximum. New +tests must cover a real-shaped multi-page Binance response, ignored/repeated +page rejection, exact `10,000` acceptance and `10,001` refusal. This is source +only: no provider request, image build, reader/bar-edge restart, Kafka, Redis, +SQLite, V1, Trading System, alpha, order or broker mutation is allowed. + +**Phase B durable-history correction and full CI exit (`COMPLETE / DEV +INTEGRATED`, 2026-09-01).** The shared Binance vendor edge now walks backward in +bounded pages of at most `1,000` rows until the declared `1..10,000` final-BAR +window is complete. Every page is retried through the existing bounded fetcher +and rejected when it is incomplete, crosses the requested close boundary, +conflicts on an open time, makes no backward progress, or creates an interval +gap. The stable edge and its deployed Compose configuration now expose the +same `10,000` bound for both bootstrap and catch-up; existing OKX bounded +pagination was left unchanged. This is a shared adapter correction, not an +alpha-specific workaround and does not add a symbol worker, topic, service or +image. + +The isolated source matrix passed `32/32` BAR-history/bootstrap tests, +including ten-page Binance history, ignored-cursor rejection, exact `10,000` +acceptance, `10,001` refusal, continuity and checkpoint recovery. The complete +CI-equivalent suite then passed `1,275` tests with `6` documented skips in +`274.128s`, inside `qdl-phaseb-full-suite`: network disabled, source mounted +read-only, non-root UID, and tmpfs-only cache/log paths. No provider was +contacted and no Data Layer role, V1, Kafka, Redis, SQLite, Trading System, +alpha, execution session, order, broker or persistent state changed. The +scoped test container is the only disposable runtime artifact and is removed +immediately after this journal entry; no image was built and no broad cache +cleanup is authorized in this slice. + +This closes the source correctness defect that prevented config-derived alpha +bindings with `maxlen` above `1,000`. Tested source commit `42f966c` +(`fix(v2): align durable bar history with alpha warmup`) was fast-forwarded to +remote `data_layer:dev`; `main` and every runtime role remain unchanged. The +next allowed operation is the release/bundle work defined by Phase C. + +**Scoped test cleanup (`PASS`, 2026-09-01).** Removed only the stopped +`qdl-phaseb-full-suite` container after its exit code and logs were recorded. +Docker inventory changed from `63` to `62` containers; images stayed +`66.15 GB`, BuildKit cache stayed `14.39 GB`, and no volume/network/runtime +resource was removed. The retained active and rollback image set was not +altered; no broad prune was performed. + +### Five-Phase Alpha Rollout - Phase C Candidate 3c5099c + +**Status:** `IN PROGRESS / BUILD-BUNDLE-TEST ONLY` (2026-09-01). + +**Objective.** Promote the source-integrated Data Layer `dev` revision +`3c5099cd6a4c8b6b5eb8926842bcfd098f101127` into exactly one canonical reader +candidate image, `qdl-v2-python:2.0.0-dev-3c5099c`, and a secret-free binding +bundle under `/home/bobby/.local/state/qdl-v2/releases/2.0.0-dev-3c5099c/`. +The bundle is compiled from the actual `execution_alpha:dev` Compose/config +inventory at `10bb7eb`, not a generic five-symbol fixture. It must be +deterministic across two compiles and include only admitted secret-free route +bindings plus typed `BLOCKED` deployment records in its report. + +**Known rollback and exact later runtime scope.** The currently serving four +reader roles use `qdl-v2-python:2.0.0-dev-d619be6`, image +`sha256:e5cea2afa405188293e28fa8b1fd1a6ac22b2b62db8aa2e95efc44913b407963`: +`query_v2_1`, `query_v2_2`, `stream_v2_active`, `stream_v2_passive` in Compose +project `qdl_v2_stable_candidate`. If and only if the candidate source/image/ +bundle gates pass, a later serial rolling packet may recreate those four roles +only. It retains V1, Kafka topology and offsets, Redis, SQLite, Rust core, +ingestors, bar edge, projectors, Trading System, alpha, broker credentials and +the order path. Rollback is the generated four-role override pointing at the +recorded `d619be6` image ID; no generic "prior image" instruction is allowed. + +**Current slice gates and exclusions.** Build one image with OCI revision +`3c5099c...`; verify label, non-root UID/GID and digest; run selected +binding/release/reference/L2/query/stream source tests inside that image with +no source mount, no network and tmpfs-only write paths. Export actual alpha +requirements, compile twice against the candidate, verify inventory, +compilation and per-binding digests, and seal the release directory with +`0700` ownership. No reader role, bundle-mounted runtime, provider, alpha, +Gateway/Risk, database, Redis, Kafka, cursor, execution session, order or +broker path may change in this build-bundle-test slice. Docker cleanup retains +the active rollback image and this named candidate; all other test artifacts +are scoped and measured. + +**Reader-health completeness correction (`IN PROGRESS / SOURCE-ONLY`, +2026-09-01).** Candidate preflight proved that both query replicas answer the +typed mTLS readiness endpoint and both stream replicas answer typed dependency +health. It also exposed a Compose correctness gap: only `query_v2_1` declared +a Docker healthcheck, so `query_v2_2` and both stream replicas could appear +merely process-up during a later serial roll. Before any runtime mutation, add +the same mTLS `/health/ready` healthcheck to `query_v2_2` and mTLS +`/health/dependencies` healthchecks to the active/passive streams; the latter +is intentional because a cooperative stream standby is healthy while not +lease-ready. Add a source regression asserting all four reader roles use the +right endpoint, certificate root and `5s/3s/20` bounded cadence. This changes +no provider, binding semantics, manifest capability, V1 route, Kafka, Redis, +SQLite, Rust, alpha, Gateway/Risk or order path. The non-active `3c5099c` +image/bundle becomes disposable only after one replacement candidate passes +the same build, provenance, deterministic-compile and rendered-Compose gates; +the currently serving `d619be6` image remains the sole rollback artifact. + +**Reader-health source gate (`PASS`, 2026-09-01).** Added the missing Docker +healthchecks without changing reader topology: `query_v2_2` now probes its +local mTLS `/health/ready`; `stream_v2_active` and `stream_v2_passive` probe +their local mTLS `/health/dependencies`. The stream endpoint deliberately +accepts a healthy standby rather than incorrectly requiring lease ownership. +`tests.test_phaseb_stable_deployment` passed `26/26` in +`qdl-v2-python:2.0.0-dev-3c5099c`, network disabled, root filesystem +read-only, UID/GID `10001`, with only an ephemeral `/tmp`; the first attempt +used an invalid nested tmpfs below a read-only source mount and exited before +tests, then the corrected invocation passed. No service, data-plane resource, +provider, alpha or order path changed. The next in-scope operation is one +replacement candidate build from this tested source, followed by the existing +deterministic inventory/compiler and sealed-bundle gates. + +**Phase C replacement candidate and rolling packet (`READY / APPROVED`, +2026-09-01).** The tested source commit is +`be59ac830c30311aa5799cc0a83db596fc6079cc`, now fast-forwarded to +`origin/dev`. Its one immutable reader image is +`qdl-v2-python:2.0.0-dev-be59ac8`, image ID +`sha256:d06dd8c84fda7eda89f07ac8f9c55c1a975b68581e82adf840e3155509d99766`, +OCI revision `be59ac8...`, release label `2.0.0-dev-be59ac8`, user +`qdl:qdl`. The selected in-image, network-disabled/read-only/non-root matrix +passed `111` tests; the build had no source mount. The real Execution Alpha +inventory export is secret-free and deterministic: semantic inventory digest +`738363ab1c8a56f75a6e1b728d96dab59b7632c09023d6a28a7e3b72068e0d60`, +`93` deployments, with two candidate compiler passes byte-identical +(`17 ADMITTED`, `76 BLOCKED`, compilation digest +`1a859f933086715fca40e5efc1bd8293a9cc25a86b03a1bab3b6f224557cf2c6`). +The sealed secret-free bundle is +`/home/bobby/.local/state/qdl-v2/releases/2.0.0-dev-be59ac8/`, manifest +digest `17196a73dbdfc599ba492370e5c42b68568b3d518ae907758c7833d79650a73b`, +with `17` binding files and only the four reader-role image fields. Its +candidate Compose renders cleanly with the active runtime env and existing +C2 override. + +The approved serial runtime action is constrained to Compose project +`qdl_v2_stable_candidate` and only `query_v2_2`, `query_v2_1`, then the +currently observed stream `STANDBY`, then the currently observed stream +`READY`, using the sealed image override above. Before and after every role, +record bounded image ID, health/dependency state, restart count and RSS. Query +roles require mTLS `/health/ready=200`; streams require both dependencies +healthy and exactly one `gateway_lease=READY` plus one `STANDBY`. Stop on the +first failure. The exact rollback is the sealed four-role override to +`qdl-v2-python:2.0.0-dev-d619be6`, image ID +`sha256:e5cea2afa405188293e28fa8b1fd1a6ac22b2b62db8aa2e95efc44913b407963`, +recreating only the failed role. Excluded: V1, Kafka topology/offsets, Redis, +SQLite, Rust core, ingestors, bar edge, projectors, Trading System, alpha, +broker credentials and every order path. This normal reader rollout neither +creates provider demand nor changes authority; the later Phase D no-order +pair tests are the first alpha consumer use of the new bindings. + +**Canonical deployment-source selection (`APPROVED / PRE-ROLL`, 2026-09-01).** +The four-role candidate must not be launched from this feature worktree. Its +new healthchecks live in the `dev` Compose source, while the sealed bundle +deliberately changes only image fields. Before the reader roll, the one +canonical Data Layer checkout `/home/bobby/data_layer` is fast-forwarded from +clean `main` to clean `origin/dev` at `be59ac8`; `main` remains the stable +release ref and is neither merged nor mutated. This is a source selector only, +not a service action. It gives the eventual containers the canonical working +directory and the exact source/image/bundle tuple, rather than a feature +worktree label or an untracked hand-written override. Returning the canonical +checkout to a released `main` revision remains part of Phase E closeout after +an approved release merge. + +**Phase C reader runtime handoff (`PASS / DEV PRE-RELEASE`, 2026-09-01).** +Using only `/home/bobby/data_layer` at `dev@be59ac8`, the sealed image override +serially recreated `query_v2_2`, `query_v2_1`, the observed standby +`stream_v2_passive`, then the observed leader `stream_v2_active`. Each query +returned typed mTLS `READY`; final Docker health is `healthy` for all four and +every role has restart count `0`. The cooperative post-roll state is exactly +one lease holder (`stream_v2_passive`, `READY`, epoch `10`) and one standby +(`stream_v2_active`, `STANDBY`); all four use image ID +`sha256:d06dd8c84fda7eda89f07ac8f9c55c1a975b68581e82adf840e3155509d99766`. +Measured post-roll RSS is bounded at query-1 `124.9 MiB`, query-2 `114.5 MiB`, +stream-active `109.1 MiB`, stream-passive `111.4 MiB` of `512 MiB` each. The +container labels resolve only canonical source files plus the sealed release +override, never a feature worktree. V1, Kafka topology/offsets, Redis, +SQLite, Rust, ingestors, bar edge, projectors, Trading System, alpha and every +order path remained untouched. This is a DEV pre-release reader rollout, not +a stable-main release and not an alpha acceptance. + +**Phase C scoped cleanup (`READY`, 2026-09-01).** Before cleanup Docker held +the active candidate `be59ac8`, active rollback `d619be6`, and the superseded +non-active build-only candidate `3c5099c` with its matching secret-free +release bundle. Before removal, require a zero-container-reference check for +the old image. Remove only that old image, old release directory, and the +caller-owned `/tmp/qdl-phasec-reader-be59ac8.*` compiler inputs/outputs after +their hashes are recorded above. Do not prune BuildKit broadly: it contains +shared layers whose exact ownership cannot be proven from this phase. Record +post-cleanup image/cache/disk values and retain the active plus one rollback +image. + +**Phase C durable-projection recovery (`IN PROGRESS / MINIMAL RUNTIME REPAIR`, +2026-09-01).** Post-roll inventory found all three existing projectors stopped +with `OOMKilled=true`: replicas 2/3 had exited about three hours before the +reader roll; replica 1 exited while the reader handoff was in progress. Their +last bounded logs show the known failure chain `no active stable stream gateway +accepted canonical data`; the old stream startup path was unavailable, then +the projector retry loop exhausted its `512 MiB` cgroup. This is a prerequisite +for real V2 alpha evidence, not an alpha strategy issue. The new stream pair +is now healthy and has a single clean lease, so recover with the narrowest +possible action: serially `start` the existing stopped containers +`projector_v2`, `_2`, `_3` in place on their current certified bounded image +`qdl-v2-python:2.0.0-phase533-projector-631d694` +(`sha256:288e617c7ec137dfcac5e94a5977772f2e0f39eac3ff48585980c0b26b0071bb`), +with unchanged `512 MiB` cgroup, `stable-projector-v1` group, offsets, TLS, +runtime/state mounts and `2048` records / `32 MiB` pending bounds. Observe +each before starting the next: running, `OOMKilled=false`, restart `0`, +healthy stream ingestion and bounded RSS/error scan. On failure stop only +that recovered role; do not recreate, reset Kafka, flush Redis, delete +SQLite, change image, V1, Rust, ingestor, bar edge, Trading System, alpha or +order path. Normal canonical/cache writes after a healthy start are expected +durable projection recovery, not a test-order or authority mutation. + +**Phase C durable-projection recovery (`PASS`, 2026-09-01).** The existing +projectors were started serially in place, never recreated: `_1` first, then +`_2`, then `_3`. After a bounded catch-up window all three are `running`, +`OOMKilled=false`, restart `0`, and their actual local HTTP readiness endpoint +returns typed `READY` with authority `RUST_PRIMARY` and config revision +`phasec36-reference-l2-r14`. The initially attempted HTTPS probe correctly +failed with `WRONG_VERSION_NUMBER`; projector health is deliberately an +internal HTTP listener, whereas query/stream public reader probes remain +mTLS. Post-recovery RSS is bounded at `_1` `116.3 MiB`, `_2` `297.5 MiB` and +`_3` `327.1 MiB` of their unchanged `512 MiB` limits. The bounded 90-second +error scan found no `error|exception|oom|killed|fatal|gateway|cache|traceback` +match after recovery. This restored normal durable projection from the +already-running stream without offset reset, Redis flush, SQLite deletion, +image change, V1/Rust/ingestor/bar-edge, Trading System, alpha or order-path +mutation. + +**Phase C cleanup and close (`COMPLETE / DEV PRE-RELEASE`, 2026-09-01).** +The current reader runtime is the canonical tuple: source +`/home/bobby/data_layer` `dev@be59ac8`; sealed bundle manifest +`17196a73dbdfc599ba492370e5c42b68568b3d518ae907758c7833d79650a73b`; +four reader roles on image +`qdl-v2-python:2.0.0-dev-be59ac8` / ID +`sha256:d06dd8c84fda7eda89f07ac8f9c55c1a975b68581e82adf840e3155509d99766`; +and named rollback `qdl-v2-python:2.0.0-dev-d619be6` / ID +`sha256:e5cea2afa405188293e28fa8b1fd1a6ac22b2b62db8aa2e95efc44913b407963`. +After a zero-container-reference check, removed only the superseded +`qdl-v2-python:2.0.0-dev-3c5099c` image, its `440 KiB` release directory and +the `724 KiB` caller-owned compiler directory. Docker images moved from +`67.47 GiB` to `66.81 GiB`; BuildKit is `15.74 GiB` with `6.876 GiB` +reclaimable and was deliberately not broadly pruned because ownership is +shared. Host disk is `204/290 GiB`, with `86 GiB` available. No active image, +volume, network, source, runtime state, V1 role or rollback image was removed. + +Phase C is therefore complete as a **DEV pre-release reader/bundle rollout**. +It does not certify any alpha strategy or paper execution. The next permitted +step is Phase D: temporary paired Binance USD-M/OKX Swap no-order proof using +real V2 data and the sealed bindings, with exact pre/post zero-mutation +evidence for each alpha class. + +### Phase 54 - Approved Full Five-Phase Alpha Consumer Execution + +**Status:** IN PROGRESS / OWNER-APPROVED, 2026-09-01. + +The owner approved execution of the complete five-phase rollout recorded in +this ledger and the cross-repository Trading System/Alpha journals. The +rollout starts with source convergence and ends only after paired real-data +no-order evidence, bounded paper canaries, release reconciliation and scoped +cleanup. No phase may silently reduce the declared target to a generic +five-liquid manifest. + +**Cross-venue invariant.** Every representative alpha acceptance runs native +Binance USD-M and OKX Swap readers concurrently, with distinct identities, +cursor/state/log namespaces and native instrument mappings. V1/V2 output +parity is compared only inside one venue and one identical candle corpus; +Binance and OKX signals are never asserted equal. A reviewed Binance alpha +without a production OKX deployment receives only a declared temporary +OKX no-order probe, never an invented paper/live strategy deployment. + +**Runtime and release invariant.** Data Layer rolls only the shared +query/stream reader roles in its release step. V1, Kafka topology/offsets, +Redis, SQLite, Rust ingest/projector, broker/order paths and DNSE/VN +V1-primary routing remain untouched unless a later named packet explicitly +records otherwise. One canonical shared image serves all bindings; phase, +symbol, interval and alpha names are not production image identities. + +**Completion and cleanup invariant.** Every source-changing slice is tested, +committed with the owner identity and integrated into dev before its phase +closes. Main changes only through the final approved stable-release PR. +Temporary containers use --rm; test images/worktrees are removed only after +remote-dev ancestry and zero-container-reference checks. The retained set +is the active release image plus one named rollback image, with pre/post disk +evidence recorded here. + +**Phase 1 source-convergence gate (PASS / dev integration next).** A fresh +remote-ref audit proves the declared V2 branches +origin/fix/phase533-v2-query-readiness, +origin/feat/v2-alpha-config-bindings and +origin/feat/v2-stable-rust-binance-okx are already ancestors of origin/dev. +They are recorded rather than merged a second time. The dev-equivalent source +gate ran in qdl-v2-python:2.0.0-dev-be59ac8 with network disabled, read-only +source, non-root UID and tmpfs-only writes: 85 tests passed, 0 failures, 0 +errors and 1 declared skip across alpha deployment binding, sealed reader +release, stable deployment and stable edge contracts. Expected fixture logs +exercise invalid CLI input, stale BAR recovery and projector recovery; they +are asserted outcomes, not provider/runtime calls. No reader role, provider, +Kafka/Redis/SQLite, alpha, Trading System, order or broker state changed. +The next source action is to fast-forward this tested journal commit to dev. + +**Phase 1 integration closure (COMPLETE / DEV INTEGRATED).** The tested +authorization and source-gate journal is remote Data Layer dev commit +32a48e121ba31f1e4a74c3522d9bfa871c40902b. It is a non-force fast-forward +from 96219ab; main remains 9d6dfb9. The official Phase 1 source tuple is +Data Layer 32a48e1, Trading System 223db53, Execution Alpha 9cfc039 and +qdl-sdk 2.0.1. No code branch had to be replayed because the required V2 refs +were already ancestors of their respective dev lines. The sole disposable +artifact is the unreferenced Trading System test image; scoped cleanup and +canonical Data Layer dev fast-forward occur after this closeout record reaches +remote dev. + +### Phase 54 - Phase 2 Config-Derived Binding Expansion + +**Status:** IN PROGRESS / SOURCE-ONLY, 2026-09-01. + +**Scope.** Compile every actual Execution Alpha Compose deployment into either +one exact secret-free V2 binding or one typed blocked record. Inputs are the +real deployment configuration and its declarative data-requirements profile: +native venue/market/product/symbol universe, final BAR interval, maxlen and +min-bars, realtime execution context, L2/reference needs, freshness/gap and +fallback policy. Strategy code and backtest parameters are not inferred or +rewritten. + +**Current baseline.** The fresh offline inventory has 93 deployments: +DECLARED=9, DECLARED_NO_ORDER_PROBE=9 and BLOCKED=75 +(DNSE_V1_PRIMARY=8, MODE_OUTSIDE_PAPER_V2_ROLLOUT=35, +PROFILE_REVIEW_REQUIRED=32). The paired probes are native OKX Swap data-plane +proofs, not fabricated strategy or broker-paper deployments. + +**Exit, tests and rollback.** Compile the actual inventory twice byte-for-byte; +validate every generated binding with both Data Layer and Trading System +parsers; cover 2500/5000/10000 maxlen, single/multi-symbol, bracket, +grid/L2 and basis/reference profiles, malformed/cross-venue/duplicate records +and required-capability failure. Tests are no-network/read-only and write only +caller-owned tmpfs artifacts. A failure blocks the affected deployment rather +than widening a route. Rollback is a source revert of the Phase 2 commits; +no runtime bundle, provider demand, reader, Kafka/Redis/SQLite, alpha, +Trading System or order path may change in this phase. + +**Closure (`PASS / SOURCE-ONLY`, 2026-09-01).** The exact Compose/config +inventory was exported twice in isolated containers with the network disabled, +read-only source and tmpfs-only scratch state. Both renders are byte-identical: +`93` deployments, inventory SHA-256 +`738363ab1c8a56f75a6e1b728d96dab59b7632c09023d6a28a7e3b72068e0d60`; +`9` DECLARED, `9` DECLARED_NO_ORDER_PROBE and `75` typed BLOCKED. The Data +Layer compiler rendered `17` admitted bindings containing `106` exact V2 +products and `76` explicit blocks, twice with compilation SHA-256 +`1a859f933086715fca40e5efc1bd8293a9cc25a86b03a1bab3b6f224557cf2c6`. +The additional compiler block is the correct native outcome for the +Binance-specific basis profile: `CAPABILITY_UNAVAILABLE:BASIS:crypto_liquid_v2` +on its OKX Swap no-order probe. It was neither cross-routed nor represented as +zero data. + +The portable Trading System binding parser accepted all `17` compiler outputs +and all `106` product identities with the same compiler digest. Focused +regression coverage also exercised `2500`, `5000` and `10000` history rows as +admitted values and `10001` as an explicit block. The registry/profile suite +covers single-symbol directional, multi-symbol, bracket/mark, grid L2 and +basis/reference requirements, malformed/duplicate/cross-venue rejection and +missing required capability. A targeted representative-runtime source scan +found no direct Binance/OKX, Redis or WebSocket client bypass in +`adaptive_hma_cpp`, `scalping_purely`, `fib_sl_tp_strength` or `dynamic_grid`. + +No runtime bundle/image/role, provider request, Kafka, Redis, SQLite, V1, +Trading System, alpha container, cursor/state/log, database row, session, +reservation, order or broker request was created. Phase 2 is complete after +this journal commit reaches `dev`; Phase 3 alone may build the canonical +reader/stream release tuple and roll its four shared roles. + +### Phase 54 - Phase 3 Canonical Reader/Stream Release + +**Status:** IN PROGRESS / OWNER-APPROVED RUNTIME PACKET, 2026-09-01. + +**Goal and scope.** Build one immutable, shared Data Layer reader image from +the Phase 2 `dev` revision and seal the actual config-derived binding bundle. +Use canonical product naming `qdl-v2-python:2.0.0-dev-`; no phase, +alpha, symbol or interval identity is allowed in the image name. After +source/image/bundle gates pass, serially recreate only `query_v2_2`, +`query_v2_1`, `stream_v2_passive` and `stream_v2_active` in the existing +`qdl_v2_stable_candidate` project. The new image serves all declared bindings. + +**Frozen blast radius.** Retain V1 `data_layer_service`, Kafka brokers/topics/ +offsets, stable Redis, SQLite/spool, Rust cores, ingestors, bar edge, +projectors, TLS/identities, Trading System, alpha processes and all order +paths unchanged. Normal reader cache reads are expected; this phase creates no +provider demand, no alpha cursor/state/log and no consumer or order record. + +**Tests and exit.** The candidate must be built without a source bind, carry +its exact OCI revision/release labels, and pass the config compiler/release +tests under `--network none`, read-only root filesystem, non-root UID and +tmpfs-only scratch. Seal inventory/report/bindings atomically with input/image/ +rollback SHA-256s; render Compose against the canonical checkout. For every +serial role, record before/after image ID, health/dependency state, restart +count and RSS. Queries require mTLS `READY`; streams require exactly one +`READY` lease and one `STANDBY`. Stop on first failure. + +**Rollback and cleanup.** The currently active reader image +`qdl-v2-python:2.0.0-dev-be59ac8` / ID +`sha256:d06dd8c84fda7eda89f07ac8f9c55c1a975b68581e82adf840e3155509d99766` +is the named per-role rollback with unchanged runtime/TLS mounts. Roll back +only the failed reader role. On success retain the new active image plus this +rollback image; remove only zero-reference temporary test images, temporary +compiler artifacts and this phase worktree after `dev` ancestry is proven. +No broad prune, volume/network/state deletion or source reset is permitted. + +**Build, seal and source gates (`PASS`, 2026-09-01).** The canonical checkout +`/home/bobby/data_layer@b1729675632822acd2496536bd58cff4160d6323` built one +non-root immutable image, `qdl-v2-python:2.0.0-dev-b172967` / ID +`sha256:8bb504458b0608f5e43f355c06b71a61e8f3018a854d3a81ac0f9084e9269160`. +Its OCI revision/version labels exactly match the source/release coordinate. +The image-contained no-network, read-only, non-root regression matrix passed +`43/43` (`test_alpha_deployment_bindings`, `test_phaseb_stable_deployment`, +`test_phase115_universal_release`). Expected negative-fixture output covered +invalid CLI input, stale final-BAR recovery and bounded DNSE queue fencing; +no provider was contacted. + +An isolated clean Execution Alpha `dev@8c4a27f` source export yielded the +actual `93` deployment inventory at SHA-256 +`738363ab1c8a56f75a6e1b728d96dab59b7632c09023d6a28a7e3b72068e0d60`. +The candidate compiler admitted `17` bindings / `106` native products and +blocked `76`, compilation SHA-256 +`1a859f933086715fca40e5efc1bd8293a9cc25a86b03a1bab3b6f224557cf2c6`. +The portable Trading System parser independently accepted all 17/106. The +sealed secret-free release is +`/home/bobby/.local/state/qdl-v2/releases/2.0.0-dev-b172967`, manifest +`998850767a0a732873b8cce22448dea7a41b035072d3a9c6155210d08b61aba8`; +it records no secret, runtime mutation or order action and has a rendered +four-role candidate plus exact per-role rollback override. Canonical Compose +rendered successfully before any role change. + +**Bounded reader handoff (`PASS / DEV PRE-RELEASE`, 2026-09-01).** With the +existing project/runtime/TLS mounts, serially recreated only `query_v2_2`, +`query_v2_1`, the observed standby `stream_v2_active`, then the observed owner +`stream_v2_passive`. All four now use `8bb504...69160`, are Docker `healthy`, +restart `0`, OOM false. Both query replicas return typed mTLS `READY`; the +stream pair converged to exactly one `READY` lease (`active`) and one +`STANDBY` (`passive`). A bounded ten-minute error scan found zero +`fatal|panic|oom|traceback|exception|not_ready|unavailable` matches. Post-roll +RSS was query `112.0/115.3 MiB` and stream `114.6/109.3 MiB`, each below the +unchanged `512 MiB` cgroup. V1, Kafka topology/offsets, Redis, SQLite, +Rust/ingestors/bar edge/projectors, Trading System, alpha and every order path +were not recreated or reset. + +**Cleanup and decision.** Before build the Docker image store was `66.81 GiB` +and host disk `204/290 GiB`; after build/roll it is `67.48 GiB` and +`207/290 GiB`. Retain only the active candidate and the named rollback +`qdl-v2-python:2.0.0-dev-be59ac8` / `sha256:d06dd8...99766`; no unreferenced +candidate image was created. Remove only caller-owned temporary compiler +inputs and the detached source worktree after this journal commit is integrated +into `dev`; no broad BuildKit prune is safe because its remaining cache has +shared ownership. Phase 3 is complete as a dev-pre-release reader rollout. +Phase 4 is the first permitted alpha/Gateway/Risk runtime proof. + +### Phase 54 - Phase 4 Paired Native Alpha No-Order Acceptance + +**Status:** IN PROGRESS / OWNER-APPROVED BOUNDED RUNTIME PROOF, 2026-09-01. + +**Goal.** Prove that the sealed V2 reader release is consumed as real +provider-derived data by the shared alpha runtime and Trading System admission +path, without creating any execution effect. Four representative runtime +classes run as paired native workloads: Binance USD-M and OKX Swap start +concurrently for one class at a time. A pair is two temporary consumers, not +two new strategy variants, images, providers, workers or symbol-specific +services. + +**Scope and invariants.** The input is release manifest +`998850767a0a732873b8cce22448dea7a41b035072d3a9c6155210d08b61aba8` and +the actual config-derived binding inventory. Each workload uses its exact +sealed consumer binding, V2 query/stream identity, real provider-derived +cache, read-only alpha source, tmpfs-only cursor/state/log paths, +`TRADING_NO_ORDER=true`, no broker credential and no direct venue client. +The temporary Gateway is a single no-order-only reader of existing +PostgreSQL/Redis/V2 cache state; it starts no background writer and is removed +after the matrix. V1, Kafka topology/offsets, Redis/SQLite durable state, +ingestors, Rust core, bar edge, projectors, running alpha services, Trading +System services and every order path are excluded. + +**Required paired matrix.** + +| Class | Binance USD-M and native OKX Swap proof | +| --- | --- | +| closed-bar directional | final BAR warmup, FIFO append, signed-cursor replay and reconnect | +| multi-symbol signal | execution symbol plus regime symbol retain native identity and one candle boundary | +| bracket/conditional | final BAR, quote and mark reach typed BRACKET/CONDITIONAL/OCO preflight | +| grid/L2 limit | L2 snapshot, delta, reset/resnapshot and tick-normalised L2 LIMIT/OCO preflight | + +For each native venue, V1/V2 semantic comparison only uses that same venue's +captured corpus. A missing active risk profile may return a typed +`PREVIEW_REJECTED` after the required V2 cache reread; it is never +auto-created, bypassed or replaced by a cross-venue profile. + +**Exit gates.** Record route/binding hashes, finality/freshness/watermark/gap, +cursor replay/reconnect, typed intent and independent Risk cache reread, +allowed fallback-return or BLOCKED result, CPU/RAM/open connections and +container exit. Exact no-order scope checks must show no execution session, +command/journal/outbox, reservation/pending exposure, order/bracket/group, +broker attempt or persistent alpha state mutation. All temporary containers +must be absent after `--rm`; remove only the caller-owned tmpfs/log/cursor +namespace and ephemeral secret environment file. No image prune, volume +deletion, Redis flush, SQLite reset or shared-state cleanup is allowed. + +**Rollback/decision boundary.** Stop the temporary no-order Gateway and pair, +remove their scoped namespace, and retain the existing V1 route plus named V2 +reader rollback image. Stop on any V2 freshness, identity, sequence, cache +reread or zero-mutation failure. Phase 5 paper canaries are forbidden until +this complete paired matrix passes. + +**Cursor-handoff preflight repair (`IN PROGRESS / SOURCE-ONLY`, 2026-09-01).** +Source review found that the initial alpha probe checked the final BAR yielded +from `warmup_then_stream`, but did not independently prove that the server +accepts the resulting signed handoff cursor on a reconnect. This is a probe +evidence gap, not a reader, provider or route failure. The narrow shared-alpha +repair adds an exact-binding V2 bar-handoff read that obtains the sealed +warmup cursor and observes `REPLAYING -> LIVE` twice from that same signed +cursor. It does not wait for the next 5m/15m/1h close, change bar finality, add +a provider request, or expose a new Data Layer endpoint. The normal alpha +stream remains unchanged and still receives its current final warmup BAR. + +The source gate must cover valid handoff/reconnect, wrong or missing control, +timeout, exact native identity and no cross-venue route selection. It is +strictly source-only: V1, query/stream roles, Kafka, Redis, SQLite, state, +identities, Trading System, alpha processes and order paths remain untouched. +After this repair passes, the existing temporary paired packet is still the +only permitted real-provider runtime action. + +**Preflight correction (2026-09-01, no runtime mutation).** The first +read-only binding/identity inspection found a real interoperability defect: +the generated per-alpha sealed binding uses a local consumer identifier such +as `alpha..binance.paper`, while V2 workload access is granted to +the platform identities `alpha.binance.paper.stable` and +`alpha.okx.paper.stable`. The existing alpha runtime incorrectly used one +identifier for both binding verification and V2 transport authentication. The +required narrow cross-repository correction is to retain the local binding +identifier for checksum/route selection and add a separate access-consumer +identifier solely to the V2 SDK/JWT transport. It does not change Data Layer +provider admission, manifest auth, route capability or server-side policy. +No alpha, query, stream, gateway, database, Redis, cursor or order process was +started in this preflight. + +**Shared bracket-context source gate (`PASS / NO RUNTIME MUTATION`, +2026-09-01).** The paired proof found and closed the one missing execution +transport path before real-provider use: a V2 alpha bracket now carries the +same normalized `risk_context` and typed `execution_intent` through the shared +SDK, Gateway bracket schema, persisted bracket metadata and generated entry or +child leg payload. This makes `data_layer_contract=V2_PRIMARY` observable by +the existing independent Risk V2 cache reread. It is additive and legacy +bracket callers still emit no context. The isolated Trading System source +matrix passed `62/62` across typed intent, bracket, alpha SDK order-state and +no-order admission tests; Alpha Runtime passed `26/26` contract tests plus +`5/5` typed-intent tests. All runs were no-network, read-only and disposable; +no Data Layer role, provider, Kafka, Redis, SQLite, PostgreSQL, alpha state, +session, command, order, bracket, reservation or broker action changed. The +next action is the reusable source-only no-order probe; Phase 4 runtime pairs +remain pending. + +**Signed-cursor proof source exit (`PASS / NO DATA LAYER MUTATION`, +2026-09-01).** The shared alpha adapter now validates the sealed V2 BAR +warmup cursor by subscribing twice with the same server-issued token and +requiring the control boundary `REPLAYING -> LIVE` both times. This uses the +existing V2 query/stream contracts; it neither adds an endpoint nor changes +provider, Rust, Python reader, Kafka, Redis, SQLite, runtime bundle or +identity. The retained alpha image passed `7/7` exact handoff/fail-closed +facade tests, `5/5` probe tests and `45/45` V2 runtime regressions in +disposable read-only/no-network containers. No real provider request or Data +Layer runtime mutation occurred. The next allowed Data Layer action remains +the already-approved paired real-provider no-order proof. + +**Risk reread ordering dependency (`IN PROGRESS / NO DATA LAYER CHANGE`, +2026-09-01).** The paired native proof needs a Trading System no-order V2 +cache reread even if an OKX strategy profile later rejects the preview. The +pending narrow Gateway/Risk source correction changes only that read ordering +for explicitly V2-marked no-order payloads; no Data Layer endpoint, identity, +reader, provider, Kafka/Redis/SQLite state or runtime role changes. The +real-provider paired proof remains blocked until that source gate passes. + +**Risk reread ordering exit (`PASS / NO DATA LAYER MUTATION`, 2026-09-01).** +Trading System now performs its existing V2 cache read before a V2-marked +no-order mode rejection; its focused source matrix passed `63/63` with a +specific zero-write ordering test. This does not modify any Data Layer +contract, reader, provider, identity, durable state or runtime role. The +paired Binance/OKX no-order proof is now permitted against the sealed reader. + +**Phase 4 paired-reader packet (`IN PROGRESS / OWNER-APPROVED`, 2026-09-01).** +The active sealed V2 query/stream reader is read by exactly eight temporary +native alpha probes: one Binance USD-M and one OKX Swap counterpart for each +directional, multi-symbol, bracket and grid/L2 class. Every probe uses its +existing sealed binding and separate mTLS/JWT access identity; it must prove +final BAR warmup, signed cursor reconnect and any declared quote/mark/L2 path +without a direct provider connection. Data Layer roles, Kafka, Redis, SQLite, +reader image/release bundle and V1 stay untouched. Results remain insufficient +for paper promotion unless all scoped pairs finish and produce zero alpha/order +state mutation evidence. + +**First paired-reader receipt (FAILED CLOSED / ZERO MUTATION, 2026-09-01).** +The packet ran the four representative native pairs, eight read-only alpha +probes, against real V2 query/stream. Every probe rejected the first exact +latest-final-BAR with required-data-not-available; no route selected V1 +fallback. The temporary no-order Gateway exited 0 and was removed. +Before/after PostgreSQL checks across all scoped accounts were identical and +all-zero; control and market Redis scoped keys were empty and unchanged; every +temporary container and private tmpfs namespace was removed. Thus no order, +state, cache, cursor, provider or execution mutation occurred. + +**Root cause and bounded repair (IN PROGRESS / SOURCE-ONLY, 2026-09-01).** +The current V2 BAR edge mounts the narrow Phase-12 projection containing only +ten five-liquid 1m routes. The sealed real alpha inventory declares 36 +deployment BAR routes, 26 unique native physical Binance USD-M/OKX Swap +routes, including 5m, 15m, 30m, 1h and 1d. Existing catalog and the shared +Rust core already carry the capability metadata; the missing link is a +reusable compiler that turns the union of admitted sealed alpha bindings into +one bounded shared BAR-edge projection. It must deduplicate exact canonical +identity, require V2_PRIMARY plus realtime plus final BAR, retain the exact +venue/source-policy/interval mapping, and reject unbound/non-final/mismatched +routes. require_live=false for a final-BAR alpha route must not suppress +materialization: it controls consumer liveness policy, not whether the shared +edge owns a provider-confirmed close. The repair is source-only until an +explicit later packet authorizes a single bar-edge rolling recreate with a new +namespaced checkpoint. It creates no symbol worker, image, topic, Redis prefix, +consumer group or alpha-specific service. + +## Phase 54.1 - Cross-Repository Alpha V2 Production Execution Ledger + +**Status:** `OWNER-APPROVED / ACTIVE`, 2026-09-01. + +**Outcome.** Make V2 the normal, versioned data-read plane for admitted alpha +and Trading System consumers. V1 remains an explicit, observable rollback only +where the sealed route permits it. `dev` is the sole production-capable +integration line; `main` is stable release-only. Every eligible crypto alpha +is proved against both native Binance USD-M and OKX Swap data. This paired +proof verifies each venue independently; it does not assert cross-venue price +or signal equality. + +**Global invariants.** One shared provider/ingest/core/bar-edge/read plane +serves all admitted symbols and intervals. No alpha, symbol, interval, feed or +venue may create a dedicated image, topic, worker, identity or long-lived +container. Requirements are declarative at the deployment/config boundary: +native venue/product/instrument, final BAR/history `maxlen`/`min_bars`, +TRADE/QUOTE/MARK/BOOK/reference feeds, freshness/finality/gap/fallback policy, +and required versus optional semantics. Provider values are real or durably +captured provider bytes; absence is typed `BLOCKED` or optional-unavailable, +never zero-filled or cross-venue substituted. DNSE/VN stays V1-primary. + +| Phase | Data Layer deliverable and hard exit | Runtime / rollback boundary | +| --- | --- | --- | +| A | Converge tested Phase-4 source with Trading System and Execution Alpha into each remote `dev`; record the three SHA tuple and `qdl_sdk` provenance. | Source-only; revert coherent `dev` commits. Do not alter dirty canonical checkouts. | +| B | Compile every actual eligible alpha configuration to a sealed native binding or typed `BLOCKED`; compile one shared BAR-edge projection from the union of admitted alpha demand **and active baseline demand**. | Source-only; no provider/runtime state. A later BAR-edge handoff must retain existing routes. | +| C | Build one canonical semver-plus-SHA reader image and sealed bundle; separately apply the bounded BAR-edge projection required for authentic final BAR. | Reader roll is only query/stream. BAR edge, if required, is one named role with its own checkpoint/config rollback; V1/Kafka/Redis/SQLite/core/ingestors/projectors stay intact. | +| D | Run bounded native Binance/OKX no-order proof for representative closed-bar, multi-symbol, bracket/conditional and grid/L2 classes; cover all admitted binding classes with data-only contract probes. | Temporary `--rm` consumers and no-order Gateway only; exact DB/Redis/state scope must remain zero-mutated. | +| E | Promote passing paper canaries per admitted class, publish stable release only after evidence, then clean merged worktrees and unreferenced test images by exact inventory. | Paper scopes are namespaced; V1 is retained rollback. `dev -> main` only through release PR/tag. | + +**Mandatory test matrix.** Source gates cover deterministic double render, +native identity isolation, malformed/duplicate rejection, required/optional +capability semantics, `maxlen` 2,500/5,000/10,000, single/multi-symbol, +bracket, grid/L2 and basis/reference declarations. Runtime gates cover final +BAR, warmup/FIFO append, signed cursor replay/reconnect, quote/mark/L2 +sequence/reset/resync, freshness/watermark/gap, bounded resource use, and +V1-fallback-return only where policy permits it. Alpha/V1-V2 parity is within +the same venue and candle corpus. No-order evidence additionally proves zero +execution sessions, commands, reservations, orders, groups, brackets, fills, +broker attempts, persistent alpha state and scoped Redis mutation. + +**Lifecycle and cleanup.** Each coherent tested source slice is committed with +the owner identity, pushed/merged to `dev`, then verified with +`git merge-base --is-ancestor` before its worktree or local branch is removed. +Never reset or delete a dirty canonical checkout. At phase close retain only +the active runtime digest and named rollback digest; record exact container +references plus disk before/after before removing unreferenced test images or +BuildKit cache. Runtime state, volumes, networks, source, secrets and active +containers are excluded from cleanup. + +**Phase B shared BAR-demand compiler exit (`PASS / SOURCE-ONLY`, 2026-09-01).** +The compiler now accepts a repeatable set of sealed alpha bindings and an +optional active `MATERIALIZED` baseline projection. The retained projection is +not trusted as a broad catalog: its schema/status/digests/binding IDs are +validated, then every retained route is resolved again through the current +strict catalog and must still be a final BAR. This prevents a new alpha union +from deleting the active Trading System 1m baseline while still rejecting an +unknown or non-final retained route. It produces one shared catalog/acquisition +projection, deduplicated by canonical identity, without creating any +symbol-specific topology. + +An isolated non-root, network-disabled, read-only test container passed `7/7` +projection tests, including alpha `require_live=false`, duplicate alpha route, +active-baseline retention and missing-baseline rejection; the existing binding +compiler/release suite passed `10/10`. A separate offline tmpfs-only real +artifact compile read all `17` sealed alpha bindings: their demand selected +`25` native final-BAR routes. Unioning the active ten-route +`trading-system.paper.stable` projection produced exactly `35` routes, with +no baseline removal. A second offline verifier mapped all 35 binding IDs to +current catalog source IDs and confirmed each exists in the active Rust core's +182-source configuration. These commands made zero provider requests, runtime +changes, order actions or persistent writes; all output was tmpfs-only. + +The source repair is ready for its coherent Data Layer commit and `dev` +integration. A later bounded runtime packet may materialize a namespaced +35-route projection and recreate only `binance_bar_edge`; it must retain the +current source image, V1, Kafka/Redis/SQLite, Rust core, ingestors, projectors, +Trading System, alpha and order path, with exact old projection/checkpoint +rollback coordinates recorded at execution time. + +**Phase A `dev` revalidation (`PASS / REMOTE INTEGRATION NEXT`, 2026-09-01).** +The locally fast-forwarded `dev` source, rather than a feature-only checkout, +passed the isolated read-only/no-network binding/projection/release matrix +`17/17`. Its Data Layer source coordinate is `29639f8`; Trading System and +Execution Alpha local `dev` counterparts passed their paired source gates at +`50da7e4` (`58/58`) and `80519d3` (`84/84`) respectively. No runtime service, +provider, Kafka/Redis/SQLite state, database, alpha, Gateway/Risk or order +path changed. The next action is one non-force push of all three `dev` refs, +followed by ancestry verification before any worktree cleanup. + +**Phase A integration and cleanup closure (`COMPLETE`, 2026-09-01).** Remote +`dev` now contains the tested source tuple at Data Layer `61d33a4`, Trading +System `d171e1d` and Execution Alpha `5ed4a5b`; in all three repositories +`origin/main` remains an ancestor of `origin/dev` and no main ref changed. The +three Phase-A feature branches are verified ancestors of their corresponding +remote `dev` ref. The one disposable image +`trading-system-test:alpha-v2-a475c59` had zero container references and was +removed. Docker image storage moved from `69.34 GiB` to `68.78 GiB`; host free +space rounded from `79 GiB` to `80 GiB`. Reclaimable BuildKit cache is left +intact (`8.537 GiB`) because its layers have shared ownership and no broad +prune was authorized. No volume, network, runtime state, active image, +rollback image, source, secret, service or alpha state was removed. The next +phase is Phase B source work from fresh `dev` feature branches. + +**Phase B real-config revalidation and closure (`COMPLETE / DEV INTEGRATED`, +2026-09-01).** Fresh feature worktrees started from the current three-repository +`dev` tuple: Data Layer `37eda44`, Trading System `2c92fee` and Execution Alpha +`9069fcb`. The Execution Alpha exporter rendered the real Compose/config +inventory twice with byte-identical SHA-256 +`738363ab1c8a56f75a6e1b728d96dab59b7632c09023d6a28a7e3b72068e0d60`: +`93` deployments (`9` declared Binance paper, `9` native OKX no-order probes, +and `75` explicitly blocked). The actual declarations cover directional, +multi-symbol, bracket/mark, grid/L2 and basis/reference profiles; their +required native venue/symbol universe, interval, history, feed and policy +fields come only from Compose/config selectors. + +The active reader image was used only as an isolated dependency container: +network disabled, read-only source/root filesystem, non-root host UID and +tmpfs-only scratch. It compiled the two inventory renders byte-for-byte into +the same report SHA-256 +`1a859f933086715fca40e5efc1bd8293a9cc25a86b03a1bab3b6f224557cf2c6`: +`17` admitted sealed bindings, `106` exact native V2 products and `76` typed +compiler blocks. Execution Alpha deployment-requirement regressions passed +`7/7`; Data Layer compiler regressions passed `6/6`; Trading System binding +contract regressions passed `12/12` and its independent parser loaded all +`17/106` generated identities. Tests covered the declared single/multi-symbol, +bracket, grid/L2 and basis/reference classes plus maxlen bounds, +malformed/duplicate and capability-failure cases. A test-only host Pytest cache +write was rejected by the read-only source mount and is explicitly harmless; +the test itself passed. + +The only paired native capability block remains the Binance-specific +`basis_arb_binance` OKX data probe for `BASIS` under +`crypto_liquid_v2`; it is recorded as +`CAPABILITY_UNAVAILABLE:BASIS:crypto_liquid_v2`, never substituted from +Binance or turned into zero. DNSE/VN stays V1-primary; sandbox/non-paper and +unreviewed profiles remain explicit policy blocks. No provider request, +runtime role/image build, Kafka/Redis/SQLite/PostgreSQL mutation, alpha +container/state, Gateway/Risk call, session, command, order or broker action +occurred. All `/tmp/qdl-phaseb-*` inventory/compiler artifacts and disposable +containers were removed. Phase C is now the next permitted action: seal one +canonical reader image/bundle from this `dev` state, then use its separately +recorded BAR-edge projection handoff to materialize authentic final BARs. + +**Phase 54.1 Phase D start and reference-plane boundary (`IN PROGRESS / OWNER-APPROVED`, 2026-09-01).** +The active source tuple for the bounded proof is Data Layer `dev@5edbc8c`, +Trading System `dev@77bfa7f`, Execution Alpha `dev@79b91e3`, and +`qdl-sdk==2.0.1`. The active shared projection has now caught up without a +synthetic write: the exact union is `35/35` final-BAR routes, all contiguous, +with `1,000..1,440` retained rows and no missing route. This opens Phase D's +real-provider read-only proof; it does not promote a reader, alpha, Gateway, +Risk or paper order path. + +The four native Binance USD-M/OKX Swap pairs remain directional, multi-symbol, +bracket/conditional and grid/L2. The separately declared basis/reference +profile is a data-only contract probe, because reference observations are +bounded provider reads and do not carry a stream cursor or execution authority. +Source review found the Data Layer public V2 SDK already exposes the governed +`/v2/market-data/reference:batch` contract, while the shared alpha facade does +not yet expose a binding-validated reference method. The narrow in-scope repair +is limited to the shared Execution Alpha facade/probe: map only sealed +`FUNDING_RATE`, `OPEN_INTEREST`, `LONG_SHORT_RATIO`, `TAKER_FLOW`, `BASIS` or +`CONTRACT_METADATA` routes to typed V2 `ReferenceRequirement` values, retain +provider/interval/coverage/lineage, and fail closed for absent native basis. +It creates no Data Layer service, topic, image, identity, worker, cache or +provider-specific fallback. The existing query/stream roles, V1, Kafka, +Redis, SQLite, Rust core, ingestors, projectors, Trading System and alpha +containers remain excluded until the later temporary no-order packet. + +**Phase D test and rollback gate.** Source tests must prove exact +route-to-reference mapping, interval/decimal/lineage preservation, required +typed `UNAVAILABLE`/`MISSING`, no cross-venue substitution and no attempt to +treat reference data as execution-grade. The runtime packet then uses only +existing query/stream readers and temporary `--rm` consumers; it records native +identity, response digest, provider lineage, coverage, freshness and bounded +resources. Its rollback is removal of temporary consumers and a source revert; +no durable data-plane or execution state is eligible for cleanup or mutation. + +## 54.2 Cross-Repository Alpha V2 Production Rollout Completion Charter + +**Status:** `OWNER-APPROVED / ACTIVE`, 2026-09-02. This is the authoritative +Data Layer half of the five-phase alpha-consumer rollout. It is jointly tracked +with Trading System section `54.13` and Execution Alpha's V2 migration ledger. +The target is V2-primary data consumption for reviewed paper alpha classes, +with V1 retained only through a sealed per-product fallback policy. It does +not change strategy/backtest logic, portfolio/risk configuration, broker +credentials, broker execution or a venue's native capability. + +**Execution authorization (2026-09-02).** The owner authorized the complete +five-phase rollout without per-slice approval prompts. Source work must merge +to `dev` at the end of each coherent tested slice; `main` remains stable +release-only. Every eligible crypto alpha proof and canary has a concurrent +native Binance USD-M and OKX Swap observation. This authorization does not +permit deletion of source/runtime state, a broad prune, V1 removal, or a +cross-venue data substitution; the rollback and cleanup boundaries below stay +mandatory. + +**Non-negotiable invariants.** Every crypto alpha proof runs a native Binance +USD-M and OKX Swap pair concurrently. Provider bytes must be real V2 data; +synthetic records are limited to deterministic unit/contract tests. A route +without a native provider-equivalent product is emitted as typed `BLOCKED`, +never substituted from the other venue, approximated with zero, or silently +sent through V1. DNSE/VN stays `V1_PRIMARY` in this rollout. One shared reader +and stream image serves every declared route: no image, worker or container is +created per alpha, symbol or interval. + +### Phase A — Source convergence and `dev` lifecycle + +- Reconcile the three V2 source branches against their respective `origin/dev` + heads, rerun their focused source tests on a clean integration checkout, and + record the immutable tuple `data_layer_sha`, `trading_system_sha`, + `execution_alpha_sha`, SDK version and generated-binding schema revision. +- `execution_alpha/dev` already exists; formalize it as the only + production-capable integration branch rather than creating a duplicate. + Feature branches begin from and merge into `dev`; `main` is release-only. +- Exit: each intended feature commit is an ancestor of `origin/dev`, no state, + log, secret or cache is staged, and no runtime role changes. A branch/worktree + is removable only after `git merge-base --is-ancestor origin/dev`. +- Rollback: leave the merged source in `dev`, revert with a normal reviewed + commit if necessary; never reset a dirty canonical checkout or rewrite main. + +### Phase B — Actual-config binding compiler + +- Compile from each paper-capable alpha deployment's real Compose/config: + `venue`, market/product, native symbol universe, interval, feed set, finality, + `maxlen`, `min_bars`, freshness, gap, required/optional and fallback policy. + Data requirements remain declarative infrastructure metadata, not strategy + name heuristics. +- Validate native instrument identity and compile one sealed binding per + consumer/deployment. Cover single-symbol, multi-symbol, bracket/conditional, + grid/L2 and reference/basis classes, including `maxlen` 2,500, 5,000 and + 10,000. Existing unreviewed/de-listed/unsupported deployments fail closed + with a machine-readable reason. +- Exit: every eligible crypto deployment has a hash/version/provenance binding + or an explicit typed block; generated secrets and runtime material remain out + of Git. The portable Trading System parser accepts every admitted binding. +- Rollback: discard only the temporary sealed bundle; no catalog, Kafka, Redis, + SQLite, provider or execution mutation occurs. + +**Phase A current-source gate (`PASS / DEV MERGE NEXT`, 2026-09-02).** The +current Data Layer Phase-D branch was tested before integration in an existing +immutable runtime image with the source mounted read-only, `--network none`, +non-root UID `10001`, read-only root filesystem and tmpfs-only scratch. +`python -m unittest` ran alpha-reader release, alpha workload identity, +reference batch, V2 reference and reference/L2 consumer tests: **55 passed, +1 expected ownership skip** in `6.883s`. The source diff adds only the sealed +binding readability/identity tooling and completed-period reference semantics; +no provider, runtime role, Kafka/Redis/SQLite, Trading System, alpha, database +or order-path state was changed. The coherent branch head is `d59c2cd`; merge +to `dev` is the next permitted Phase-A action. + +**Phase A integration closure (`COMPLETE / DEV`, 2026-09-02).** The fast-forward +source SHA `e7bd1db` equals `origin/dev` and was revalidated after integration: +the same isolated test matrix completed **55 passed, 1 expected skip** in +`6.290s`. `origin/main` remains an ancestor of `origin/dev`. No runtime role, +provider request, data store, alpha, Trading System, broker or order path was +mutated. The feature worktree remains retained because Phase D still uses its +active source; it is not an archival copy and is eligible for removal only in +the final exact cleanup inventory. + +### Phase C — Canonical reader/stream release artifact + +- Build and attest one canonical `qdl-v2-python:-` reader/stream + image from `dev`, with source SHA, image digest and bundle revision recorded. + Replace phase-named test identity only for `query_v2_*` and `stream_v2_*`. + Rust is rebuilt only if Rust source changes. +- Render and validate V2-primary plus V1 fallback routing. Rolling affects only + those named reader roles; V1, Kafka topology/offsets, Redis, SQLite, + ingestors, Rust core, projectors, Trading System, alpha and order path remain + unchanged. Retain the exact old reader image and bundle for rollback. +- Exit: real V2 final BAR, quote/trade, mark/index, L2 and reference routes are + reachable only through their sealed policy. No role has a phase-worktree name + as its production identity. + +### Phase D — Paired real V2 no-order proof + +- Run one Binance/OKX pair at a time, concurrently within the pair, for four + representative alpha classes: `adaptive_hma_cpp` closed-bar, + `scalping_purely` multi-symbol BTC-regime/ETH-execution, + `fib_sl_tp_strength` bracket/conditional, and `dynamic_grid` grid/L2. Run + `basis_arb_binance` as a separate data-only reference capability probe. +- Each temporary consumer uses real V2 data, a sealed native binding, + `V2_PRIMARY`, no broker credentials, tmpfs cursor/state/log directories and + `--rm`. Prove warmup/FIFO append, final BAR, signed cursor replay/reconnect, + native identity isolation, quote/mark/L2 sequence or reset/resync as required, + and bounded CPU/RAM/feed age/watermark/gap/fallback counters. +- Alpha intent must reach the temporary no-order Gateway/Risk preview unchanged; + Risk independently rereads V2 execution context. Before/after exact scoped + PostgreSQL and Redis checks must show zero sessions, commands, reservations, + orders, fills, groups, brackets, broker attempts and persistent alpha state. + V1/V2 signal/`pos_weight`/sizing/intent parity is compared only within the + same venue/candle corpus, never between native venues. +- Exit: all admitted pair products pass. Any unavailable OKX reference product + is a tested typed block; it cannot be hidden by a fallback or an invented + cross-venue value. Rollback is removal of only the temporary consumers and + their namespaced tmpfs/evidence artifacts. + +### Phase E — Paper canary, release closure and hygiene + +- Promote one passing class at a time to a namespaced natural closed-candle + paper canary. Binance runs end-to-end when its paper broker adapter is + available. OKX runs a matching paper canary only when its paper broker + capability is enabled; otherwise it remains explicitly V2 no-order certified, + never mislabeled as execution-ready. +- Validate V2-primary read, policy-allowed forced V1 fallback-return, strategy + input/signal/`pos_weight`/sizing parity before submit, and a bounded full + paper lifecycle with no unexpected stale or fallback. Render-validate every + reviewed alpha deployment against its sealed binding. +- Exit: source commits are merged to `dev`, evidence is journaled in all three + plans, and a release PR is the sole route to `main`. Retain active plus one + named rollback image. After container-reference and disk checks, remove only + merged feature worktrees, stopped disposable smoke containers, exact + unreferenced test images and their unused build cache; never prune volumes, + networks, runtime state, active containers, secrets or user source. + +**Phase B actual-config revalidation (`PASS / SOURCE-ONLY`, 2026-09-02).** A +disposable, network-disabled Execution Alpha runtime exported the current real +Compose/config inventory to a namespaced temporary artifact. It produced `93` +deployments with inventory SHA-256 +`7e27bebc8cd82113d78cda0cfaed9759f32c54fbb10bfe83b795ecc08d1b2c5c`. +The Data Layer's actual `compile_alpha_deployment_bindings.py` algorithm then +resolved it against the current source catalog, reference/L2 entitlement, +release routing and policy. It wrote `18` admitted sealed bindings and `75` +typed blocks, compilation SHA-256 +`aef316653ecfb1423c89fcdeec145faf3fd249243a37c849e65681267d8d04e4`. +The additional admitted binding is the native OKX basis companion limited to +funding and contract metadata; unavailable native basis/ratio/OI/taker feeds +remain optional typed blocks. No broad BTC/ETH-only manifest, cross-venue +substitution, provider request, runtime bundle, service, Kafka/Redis/SQLite or +execution mutation was used. + +The portable Trading System parser loaded every one of the `18` generated +bindings with exact consumer and release-manifest identity. Data Layer compiler +and reader-release unit suites passed `10/10` in a read-only, non-root, +network-disabled container. The temporary artifact namespace is retained only +through Phase C/D because it is the exact input to the candidate sealed reader +release; it will be removed after its digest has been recorded and the release +bundle is independently verified. Phase B is therefore complete as source +evidence; Phase C may build one canonical reader artifact from this immutable +input. + +**Phase B Python 3.10 compatibility repair (`COMPLETE / SOURCE-ONLY`, +2026-09-02).** The published Python `3.10+` support is now internally +consistent: one `qdl._compat.StrEnum` preserves the string wire behavior used +by canonical enums, all affected core modules import that shared primitive, and +the release/SBOM reader uses standard-library `tomllib` on 3.11+ with the +conditional locked `tomli` dependency on 3.10. No enum value, V2 schema, +provider adapter, Docker role, bundle, runtime routing or data-plane state was +changed. + +**Compatibility evidence and cleanup.** A disposable Python 3.10 container +installed only declared import dependencies into a caller-owned temporary +directory; its actual proof then ran `--network none`, read-only, non-root and +with tmpfs-only output. `tests.test_python310_compat` passed `3/3`; two real +Compose/config exports were byte-identical and two Data Layer compiles were +byte-identical. The independent Trading System parser accepted all `18` +bindings. The verified output was `93` deployments, `18` admitted bindings, +`75` explicit typed blocks, inventory SHA-256 +`7e27bebc8cd82113d78cda0cfaed9759f32c54fbb10bfe83b795ecc08d1b2c5c` and +compilation SHA-256 +`aef316653ecfb1423c89fcdeec145faf3fd249243a37c849e65681267d8d04e4`. +Python 3.12 regression passed `55/55` with one expected ownership skip, and +the compiler/release suite passed `36/36` (negative-case stderr was expected +by those tests). All temporary dependency/inventory/binding directories and +containers were removed. The sole pulled test image `python:3.10-slim` had no +container references and was removed; host root usage changed from +`248523239424/310911414272` to `248521027584/310911414272` bytes. Rollback is +a normal source revert; no runtime change is allowed in this slice. + +**Phase C exact-rollback correction (`IN PROGRESS / SOURCE-ONLY`, +2026-09-02).** Read-only runtime provenance found the four approved reader +roles are intentionally not on one image: `query_v2_1` and `query_v2_2` are +on `qdl-v2-python:2.0.0-a3f423d` +(`sha256:dccc24b2f4a9f4f0aff58551cfe09f7ce8e7a5f42e4ef372b27b0c5b6b58d659`), +while `stream_v2_active` and `stream_v2_passive` are on +`qdl-v2-python:2.0.0-5edbc8c` +(`sha256:b2a2848011c84a6f6eaafd74b5a1a5b4908e0c1f9cfdbeb78325c3c0c3a0985c`). +The old reader-release generator can encode only one rollback image for all +four roles, so it cannot make an exact recovery promise for this real state. +Before any new reader build, change only the secret-free release generator, +its unit tests and this runbook to require a complete per-role +`service -> {image_reference, image_id}` rollback mapping, validate every +service and immutable ID, and render the exact per-role rollback override. +The new versioned bundle manifest must record that mapping and reject an +incomplete/mutable/candidate-equal map. This is control-plane provenance only: +no V1 endpoint, provider adapter, Kafka/Redis/SQLite state, Rust core, +ingestor, projector, query/stream role, Trading System, alpha or order path +changes in this correction. Source exit is deterministic bundle tests plus +rendered mixed-image override proof; rollback is a normal source revert. + +**Phase C canonical reader packet (`APPROVED / FOLLOWS SOURCE EXIT`, +2026-09-02).** After that correction is committed to `dev`, fast-forward the +canonical checkout and build exactly one shared image from the resulting full +source SHA as `qdl-v2-python:2.0.0-`. Seal one versioned release bundle +from the existing actual inventory/compiler digests +`7e27bebc...d1b2c5c` / `aef31665...d8d04e4`; render it only as the final +image override together with the exact mixed rollback override. The sole +eligible rollout roles are `query_v2_2`, `query_v2_1`, then the observed +stream standby and lease holder. V1, Kafka topology/offsets, Redis, SQLite, +Rust, ingestors, bar edge, projectors, Trading System, alpha and all order +paths remain excluded. Per-role health/provenance/lease checks gate the next +step; rollback recreates only the failed role with its recorded pre-roll image +and unchanged runtime mount. + +**Phase C exact-rollback source gate (`PASS / SOURCE-ONLY`, 2026-09-02).** +`prepare_alpha_reader_release.py` now emits the versioned +`qdl.v2.alpha-reader-release.v2` control-plane manifest. Its candidate and +rollback overrides use digest-pinned `tag@sha256` selectors, and every bundle +requires exactly `query_v2_1`, `query_v2_2`, `stream_v2_active` and +`stream_v2_passive` with their own canonical tag/immutable ID pair. Missing, +duplicated, malformed or candidate-equal entries fail closed. The unit suite +passed `10/10` for compiler/release behavior, including a deterministic +mixed-query/stream rollback override and all reject paths. A separate actual +inventory CLI dry run as the operator UID, network-disabled and read-only, +accepted the current mixed map with `18` bindings and manifest SHA-256 +`5bdcfb7c917ccc380773a4e2d5ca5e194308288188291ed88ffca5173b43e7ae`; +the temporary output was tmpfs-only and no release bundle was written. The +initial UID-10001 attempt correctly could not traverse the operator-only input +directory, confirming that bundle generation remains an operator action rather +than a reader privilege. No source provider, container, Kafka/Redis/SQLite, +V1, Trading System, alpha or order-path state changed. This source slice is +ready to commit to `dev`; only the subsequent packet may build or recreate the +four reader roles. + +**Phase C canonical runtime packet (`APPROVED / PRE-BUILD`, 2026-09-02).** +The canonical checkout is now Data Layer `dev@3e3b79285426772c5da774224a28006b47642b4c`. +Build exactly one shared Python reader/stream image from that checkout as +`qdl-v2-python:2.0.0-3e3b792`; verify OCI revision/user/ID, then run the +reader-release and reference/L2 contract subset from the immutable image with +no source mount, `--network none`, read-only root and tmpfs-only test state. +Regenerate the real Execution Alpha Compose/config inventory and compilation +twice in disposable containers, then seal one atomic operator-owned release +under `/home/bobby/.local/state/qdl-v2/releases/2.0.0-3e3b792` from those exact +digests. Its candidate selector is digest-pinned. Its exact rollback mapping +is query 1/2 to `2.0.0-a3f423d` / +`sha256:dccc24b2f4a9f4f0aff58551cfe09f7ce8e7a5f42e4ef372b27b0c5b6b58d659` +and stream active/passive to `2.0.0-5edbc8c` / +`sha256:b2a2848011c84a6f6eaafd74b5a1a5b4908e0c1f9cfdbeb78325c3c0c3a0985c`. +Only after source/image/bundle preflight passes may Compose serially recreate +`query_v2_2`, `query_v2_1`, then the observed stream standby and leader. Every +role must show its pinned candidate selector, expected provenance, zero +restart/OOM and correct ready/lease state before continuing. A failure rolls +back only that role with its own recorded selector and unchanged runtime mount. +V1, Kafka topology/offsets, Redis, SQLite, Rust, ingestors, bar edge, +projectors, Trading System, alpha and all order paths are excluded; normal +market-data writes are not introduced by this reader-only packet. + +**Phase C bounded reader packet (`OWNER-APPROVED / PRE-BUILD`, 2026-09-02).** +The candidate is one shared image tag `qdl-v2-python:2.0.0-5edbc8c`, built +from Data Layer `dev@5edbc8c7707be3a0f57117d7b78bc6d7f0f8e89a`; the exact +candidate digest will be sealed before any recreation. The only roles eligible +for rolling recreation are `query_v2_1`, `query_v2_2`, `stream_v2_active` and +`stream_v2_passive`, currently all on immutable +`sha256:d6f15b186ebd5916b19847685487a63ddbfd1eef844cecfcedd11858186448b5` +(`qdl-v2-python:2.0.0-dev-5edbc8c`). The bundle is generated from the exact +inventory/compiler artifacts in `/tmp/qdl-phase54-alpha-bindings.UYpveN` and +is written atomically under the versioned QDL release state root. + +No other named role is in blast radius: V1, Kafka topology/offsets, stable +Redis, SQLite, Rust core, ingestors, bar edge, projectors, runtime/TLS mounts, +Trading System, alpha containers and every order path are excluded. Rollback is +the existing four-role image digest and current reader override only; it does +not reset, flush, delete or rebuild durable state. Post-roll only verifies +health, sealed binding identity and real provider reads; no execution request +is allowed. + +**Phase C artifact and reader handoff (PASS / DEV PRE-RELEASE, 2026-09-02).** +One canonical reader/stream image was built from Data Layer dev source +5edbc8c7707be3a0f57117d7b78bc6d7f0f8e89a as qdl-v2-python:2.0.0-5edbc8c, +image sha256:b2a2848011c84a6f6eaafd74b5a1a5b4908e0c1f9cfdbeb78325c3c0c3a0985c. +The sealed release root is +/home/bobby/.local/state/qdl-v2/releases/2.0.0-5edbc8c; its secret-free +binding manifest SHA-256 is +33488998c7c890e70ec30833d3455cd7f51e018681618ce470f3b6d829d84fa8. +The four approved reader roles were recreated serially and are healthy: +query_v2_1, query_v2_2, stream_v2_active, and stream_v2_passive. Their exact +retained rollback is qdl-v2-python:2.0.0-dev-5edbc8c +(sha256:d6f15b186ebd5916b19847685487a63ddbfd1eef844cecfcedd11858186448b5). +V1, Kafka topology/offsets, Redis, SQLite, Rust core, ingestors, bar edge, +projectors, Trading System, alpha and every order path were not changed. +Image label/import smoke passed. This is a reader artifact handoff only; it +does not yet certify a real alpha consumer. Phase D is the next active gate. + +**Phase D reference-period semantics correction (`IN PROGRESS / SOURCE-ONLY`, +2026-09-02).** The first authenticated, disposable alpha reference probe +proved V2 identity, mTLS/JWT and sealed-route admission, then exposed a +semantic freshness defect in the Binance USD-M adapter: the official provider +defines `takerBuySellVol.timestamp` as the *start* of its requested period. +The adapter already used that fact for pagination coverage, but emitted the +same start timestamp as `ReferenceObservation.observed_at_ns`; the V2 query +freshness policy therefore treated a completed daily row as almost one whole +period older than it is. Native Binance basis uses the same period-start +semantic and must follow the same contract. This is not an SLA change, a +fallback, a provider retry, or a synthetic-data exception. + +The source correction is deliberately shared and bounded: preserve raw +provider period start/end in observation labels; expose `observed_at_ns` as the +verified period close for freshness/finality; and make pagination selection, +cadence and coverage operate on the raw period-open coordinate. Contract tests +must prove raw lineage preservation, exact close-time freshness, duplicate and +gap detection, and unchanged period-start coverage for both TAKER_FLOW and +native BASIS. The only subsequent runtime packet eligible to exercise the +fix is a serial recreation of the existing `query_v2_1` and `query_v2_2` +reader roles with one immutable Python image built from this source revision; +the currently active reader digest remains the exact rollback. V1, Kafka +topology/offsets, Redis, SQLite, Rust core, ingestors, bar edge, projectors, +Trading System, alpha services and every order path remain excluded. + +**Phase D reference-period source gate (`PASS / SOURCE-ONLY`, 2026-09-02).** +`BinanceUsdmReferenceAdapter` now keeps `period_open_time_ms`, +`period_close_time_ms` and `timestamp_origin=PROVIDER_PERIOD_START` on native +TAKER_FLOW and BASIS observations. Their canonical observation time is the +verified close; the shared paginator separately uses the preserved open-time +coordinate for provider windows, cadence, duplicate detection and coverage. +Thus no raw provider fact is rewritten and a one-period freshness correction +cannot hide a history gap. An immutable, non-root, read-only, network-disabled +container ran `tests.test_phase104_reference_batch`, +`tests.test_phase113_reference_v2` and +`tests.test_reference_l2_consumer_acceptance`: `48/48 PASS` in 6.589s. +`git diff --check` and changed-file `py_compile` passed. No provider call, +container service, Kafka/Redis/SQLite write, V1/Trading System/alpha action or +order-path mutation occurred. The source is ready for a coherent commit and +then the separately bounded two-reader image/runtime packet. + +**Phase D reader-only candidate packet (`APPROVED / PRE-BUILD`, 2026-09-02).** +Build one shared reader image from source commit `021b372` as +`qdl-v2-python:2.0.0-021b372`. Verify its OCI source/release labels and run +the same immutable source-contract subset before runtime use. If and only if +that gate passes, atomically render a temporary reader override and serially +recreate only `query_v2_1` then `query_v2_2`; their current exact image +`qdl-v2-python:2.0.0-5edbc8c` / digest +`sha256:b2a2848011c84a6f6eaafd74b5a1a5b4908e0c1f9cfdbeb78325c3c0c3a0985c` +with the unchanged runtime mount is rollback. The candidate must become +healthy and retain the sealed identity before one real read-only +Binance/OKX reference pair. No stream role, V1, Kafka topology/offsets, +Redis/SQLite data, Rust, ingestion, projector, Trading System, alpha service, +database, broker credential or order path belongs to this packet. + +**Phase D reader-only candidate build (`PASS / RUNTIME UNCHANGED`, +2026-09-02).** Source commit `021b372` built the one shared candidate +`qdl-v2-python:2.0.0-021b372`, image digest +`sha256:82e0c6cc57b8b4630cfa1cfaa7f017128f332601f2ea6613595be4abfd8035e2`. +OCI labels pin revision `021b372` and version `2.0.0-021b372`. Directly from +that immutable image, a non-root/read-only/no-network/tmpfs-only contract run +again passed `48/48` reference/L2 tests in 6.859s. No service was recreated; +the active reader remains `2.0.0-5edbc8c` and its digest is retained as the +only rollback for this two-role packet. The next operation is a rendered +two-query override preflight, followed by serial recreation exactly as recorded +above; no other role is eligible. + +**Phase D two-query preflight (`PASS / RUNTIME UNCHANGED`, 2026-09-02).** The +scoped override +`/home/bobby/.local/state/qdl-v2/phase54-reference-finality-021b372/query-image.override.yml` +(SHA-256 `e9aa1ded94595800f4f60981eecaece446ff1d13539430ef5da838e1b38df6c7`) +renders only `query_v2_1` and `query_v2_2` to `qdl-v2-python:2.0.0-021b372`. +The paired rollback override SHA-256 is +`315927b39c76452746dc6ceded02a4bdc0f473706c004a201292c91b327d1b9e` and +restores only those two roles to `qdl-v2-python:2.0.0-5edbc8c`. Compose +preflight proves stream roles remain `2.0.0-5edbc8c`; Rust, both ingestors and +the projector remain on their prior image selectors. The next approved action +is serial `--no-deps --force-recreate` of query replica 1, health/provenance +verification, then replica 2. Any readiness/provenance failure triggers only +the two-role rollback override; no data-store reset/flush or topology mutation +is permitted. + +**Phase D sealed-binding workload-readability defect (`FAIL-CLOSED / SOURCE +CORRECTION APPROVED`, 2026-09-02).** Both serially recreated query replicas +are healthy on `qdl-v2-python:2.0.0-021b372`; no other role changed. The first +paired disposable Binance/OKX reference readers stopped before a V2 request, +provider call, Gateway/Risk call or order action because their sealed binding +files are generated mode `0640` and owned by host UID/GID `1001`, while the +immutable alpha image deliberately runs as non-root `alpha` UID/GID `1000`. +The binding contains only route identity/policy/hash data, never a secret; +keeping it unreadable to its intended workload makes the release artifact +non-deployable. Correct the shared release generator, not the old artifact or +the alpha image: published `bindings/*.binding.json` must be immutable +world-readable `0444` (or an equivalently readable bind mount), while release +env, JWT/private keys and TLS private keys remain private. Add a regression +that asserts the split. Regenerate a new atomic release bundle from the +candidate source, validate its hashes/modes with the actual alpha UID, then +rerun exactly the same pair. No chmod-in-place, service restart, provider +simulation, V1/Kafka/Redis/SQLite mutation, Trading System/alpha service +change or order action is authorized by this correction. + +**Phase D sealed-binding source gate (`PASS / SOURCE-ONLY`, 2026-09-02).** +`prepare_alpha_reader_release.py` now classifies artifacts by their actual +security boundary: route bindings are immutable `0444` workload-mount inputs; +release manifests, compilation reports and compose overrides remain `0640`; +private runtime env/JWT/TLS material remains outside this release bundle and is +not widened. The regression creates two atomic bundles and proves deterministic +content, `0444` for every published binding and `0640` for the release +manifest. In immutable `qdl-v2-python:2.0.0-021b372`, network-disabled, +read-only, non-root mode, `tests.test_alpha_reader_release` passed `4/4` in +1.768s. `git diff --check` and `py_compile` passed. No runtime service, +provider call, source authority, V1/Kafka/Redis/SQLite state, Trading System, +alpha service or order path changed. The next permitted operation is a new +atomic release bundle with this generator, followed by a UID-1000 readability +check and the same two disposable V2 reference probes. + +**Phase D binding-readable artifact packet (`APPROVED / PRE-BUILD`, +2026-09-02).** Build one immutable shared reader image from source commit +`136c52e` as `qdl-v2-python:2.0.0-136c52e`, run the same network-disabled +release-generator test in that image, and seal one new versioned bundle from +the existing verified inventory/compiler report. The bundle may point only to +this candidate reader image and the retained `2.0.0-5edbc8c` rollback reader. +Before any reader role is recreated, verify the exact sealed bindings have mode +`0444` and can be parsed by UID/GID `1000:1000`; otherwise stop before runtime +mutation. The only later role change remains serial `query_v2_1` then +`query_v2_2`, with per-role health/provenance checks and exact rollback. No +V1, Kafka topology/offsets, Redis/SQLite, Rust, ingestion, bar edge, +projector, Trading System, alpha service, broker credential or order path is +in scope. + +**Phase D binding-readable reader handoff (`APPROVED / PRE-ROLL`, +2026-09-02).** The new bundle is +`/home/bobby/.local/state/qdl-v2/releases/2.0.0-136c52e`, source +`136c52e6bade5bbfa9c5baf374286ee98e78826e`, candidate image +`qdl-v2-python:2.0.0-136c52e` / image ID +`sha256:2a2b533ad073d18953fab526fbfca22dc2c76d7b8d656b7cae059e8d43b48a50`, +and release manifest SHA-256 +`60670346a9ac2a9edb723d1a709f37d842b11d5dc5fe75c515d92d55c54379ad`. +All 18 binding files are mode `0444`; the alpha UID `1000` parsed an actual +sealed binding, while the release manifest and compose overrides remain mode +`0640`. The next bounded packet writes two temporary image-only Compose +overrides and serially recreates exactly `query_v2_1` and `query_v2_2`. Its +exact rollback is their current `qdl-v2-python:2.0.0-021b372` image ID +`sha256:82e0c6cc57b8b4630cfa1cfaa7f017128f332601f2ea6613595be4abfd8035e2` +with the unchanged runtime mount. The older `2.0.0-5edbc8c` remains retained as +the pre-Phase-D reader rollback; neither is removed during this handoff. No +other role or durable state may change. + +**Phase D workload-identity readability defect (`FAIL-CLOSED / SOURCE +CORRECTION REQUIRED`, 2026-09-02).** Both query replicas rolled serially to +`2.0.0-136c52e`, are Docker-healthy with restart count zero, and no other V2 +role changed. A non-network UID-1000 alpha-container check confirms the +Binance TLS identity is readable, but confirms the staged OKX client private +key is not readable (`EACCES`): it is host-owned `1001:1001` mode `0440` while +the immutable alpha runtime intentionally runs UID/GID `1000:1000`. This +stopped before any provider or V2 API request. Do not work around it by running +the alpha as the host user or making the original private key world-readable. +The required shared fix is a per-workload identity materialization path that +creates an alpha-UID-owned, private `0400` client-key mount (and read-only +certificate/CA) from the existing protected staging source, with a regression +for both Binance and OKX. It must leave the source identity private, emit no +secret value, and be reusable by every alpha/venue. Only after an actual +UID-1000 mTLS read check passes may the paired real reference probes retry. +V1, Kafka/Redis/SQLite, Rust/ingestors/bar edge/projectors, Trading System, +alpha service definitions and all order paths remain excluded. + +**Phase D workload-identity materialization slice (`APPROVED / SOURCE-ONLY`, +2026-09-02).** Add one filesystem-only Data Layer control-plane helper, not a +service and not a per-symbol artifact. It accepts exactly a protected TLS +directory (`ca.crt`, `client.crt`, `client.key`) and JWT directory +(`private.key`), validates regular-file/no-symlink inputs, and atomically +creates one per-workload output under the governed QDL state root. The mounted +TLS/JWT leaf directories are owned by the declared non-root workload UID/GID; +their private keys are `0400`, certificates/CA are `0444`, and the source keys +remain unchanged. The release/binding manifest records only hash/path/mode +metadata, never secret bytes. Unit coverage must prove deterministic dry run, +atomic materialization, source-symlink/unexpected-file rejection, private-mode +split and a real UID-1000 read. The helper must require effective root only for +apply/chown and be invoked through a bounded disposable Docker-root helper; it +must not alter any existing identity in place. On exit, materialize only the +two disposable Binance/OKX probe identities and retry the exact paired +reference read. No runtime role, provider source, durable store, Trading +System/alpha service or order path may change. + +**Phase D workload-identity source gate (`PASS / SOURCE-ONLY`, 2026-09-02).** +Added `scripts/materialize_alpha_v2_identity.py` and the Phase-54 runbook +contract. The helper accepts only the exact TLS/JWT file sets, rejects +symlinks, directories, extra/missing/empty files and an existing target, and +requires effective root only for `--apply` ownership changes. It atomically +renames a staged output under `workload-identities/`; `tls/` and +`jwt/` are `0500` and UID/GID-bound, client/JWT private keys are `0400`, and +certificates/CA/public key are `0444`. The protected source remains mode +`0440`/unchanged. The root-only filesystem test plus release generator test +passed `7/7` in 1.715s inside immutable `qdl-v2-python:2.0.0-136c52e`, +network-disabled, read-only with tmpfs-only test state. `git diff --check` and +`py_compile` passed. A separate disposable Docker bind-mount check remains the +required real UID-1000 proof because the intentionally private host state root +is not directly traversable by the alpha process. No runtime role, provider, +durable state, V1, Trading System, alpha service or order path changed. + +**Phase D two-identity materialization packet (`APPROVED / PRE-APPLY`, +2026-09-02).** Invoke the committed helper through one disposable root Docker +container per exact paper probe identity: `phase54-reference-binance` from the +existing `alpha-binance` TLS/JWT sources and `phase54-reference-okx` from the +existing `alpha-okx` TLS/JWT sources. Each helper has network disabled, +read-only root/source mounts, tmpfs `/tmp`, and only the governed QDL state +root bind-mounted writable for its new target. It may create only +`workload-identities/`; it must not mutate the protected source identity +or existing release bundle. Then a UID-1000 alpha image mounts only the two +leaf directories read-only and proves it can open both private keys without +printing them. No running service, provider request, durable state, V1, +Trading System, alpha process, Gateway/Risk or order path is included. + +**Phase D two-identity materialization (`PASS / RUNTIME UNCHANGED`, +2026-09-02).** The helper created only +`workload-identities/phase54-reference-binance` (manifest SHA-256 +`c7b0b95fc9d5533ba9ab25f2b2ec1dab35af6b746fb9801147a9b2f459c6c13d`) and +`workload-identities/phase54-reference-okx` (manifest SHA-256 +`035c8ca6f80bbf11ce720e80c76cbc3bc2a35d4a8fd429a214914baf4439bfac`). The +four protected source-key SHA-256 values were identical before/after. The host +operator cannot traverse the new workload-owned `0500` leaf directories; two +disposable non-network `execution-alpha-runtime:2.0.0-3f366a9` containers +running UID/GID `1000:1000` successfully opened both mounted TLS and JWT +private keys without printing them. No provider or V2 API request, service +recreate, durable mutation, V1/Trading System/alpha process or order action +occurred. The exact paired Binance USD-M/OKX Swap reference readers may now +retry once with the new binding bundle and these leaf mounts only. + +**Phase D reference-window and funding-boundary correction (`IN PROGRESS / SOURCE-ONLY`, 2026-09-02).** The paired authenticated no-order diagnostic reached the real V2 reference plane with the active alpha manifest revisions (`9` Binance, `8` OKX) and no broker credential or durable alpha state. It exposed two bounded semantic defects rather than a provider outage: the reusable probe applied one daily closed-BAR right edge to every reference product, while funding is an independently settled series; and Binance funding rows may arrive a few milliseconds after an official settlement boundary although the adapter's existing tolerance was only applied after the provider response had already been bounded and filtered. The correction is deliberately narrow: retain raw provider timestamps and strict full-coverage/freshness checks, use a declared settled funding window for the probe while preserving completed-period windows for taker/basis and provider snapshot windows for OI/long-short, and make the existing Binance funding tolerance effective in the outbound pagination envelope and selection boundary. No TTL is widened, no result is fabricated, no cross-venue fallback is permitted, and V1, Kafka, Redis, SQLite, Rust, ingestors, projectors, Trading System, alpha services and every order path remain excluded. Required gates: deterministic boundary/coverage and stale-history regression tests, immutable source test subset, then one concurrent real Binance/OKX reference pair; any non-OK product remains a fail-closed result. + +**Phase D Binance funding-boundary source gate (`PASS / SOURCE-ONLY`, 2026-09-02).** `BinanceUsdmReferenceAdapter` now applies its pre-existing 60-second funding-settlement tolerance to the outbound provider end boundary and the accepted right-edge selection, while retaining the provider's raw timestamp in the canonical observation. It does not relax other products or make a full-coverage claim for a row outside the declared tolerance. A deterministic regression simulates a provider that withholds the final `+3ms` funding row unless the requested end is widened; it passes only with the real envelope and selection behavior. In immutable `qdl-v2-python:2.0.0-136c52e`, network-disabled/read-only/non-root source tests `tests.test_phase104_reference_batch`, `tests.test_phase113_reference_v2` and `tests.test_reference_l2_consumer_acceptance` passed `48/48` in `7.645s`; focused adapter coverage passed `25/25` in `0.235s`. The prior paired real diagnostic remains evidence only: Binance funding had a genuine `+5ms` provider settlement jitter and the requested daily right edge omitted it. No runtime service, provider configuration, durable state, V1, Trading System, alpha or order path changed. The source is ready for a single shared query-reader image after its companion Alpha Runtime source slice commits. + +**Phase D settled-reference reader packet (`APPROVED / PRE-BUILD`, +2026-09-02).** Build exactly one shared Python reader image from committed +source `a3f423d` as `qdl-v2-python:2.0.0-a3f423d`, with OCI revision/release +labels pinned to that commit. Independently build exactly one standard Alpha +Runtime probe image from committed source `b1299bc` and the already pinned +Trading System Alpha SDK revision, as `execution-alpha-runtime:2.0.0-b1299bc`. +No Numba derivative is needed for the reference-only probe. Both builds are +sequential because the host has bounded free memory; they must be inspected +and smoke-imported as non-root, read-only, network-disabled images before any +runtime use. The only eligible runtime change after the build gates is a +serial replacement of `query_v2_2` then `query_v2_1` with the new shared +reader image through one operator-state override. Their exact rollback is the +currently active `qdl-v2-python:2.0.0-136c52e` image ID +`sha256:2a2b533ad073d18953fab526fbfca22dc2c76d7b8d656b7cae059e8d43b48a50` +and unchanged runtime mount. The packet excludes streams, V1, Kafka +topology/offsets, Redis, SQLite, Rust, ingestors, bar edge, projectors, +Trading System, alpha services, broker credentials and every order path. Its +acceptance is one concurrent disposable Binance USD-M/OKX Swap real V2 +reference probe using sealed bindings/UID-1000 workload identities; every +requested reference product must be strict-OK and no alpha durable state, +Gateway/Risk request or order may be created. Disk baseline before this packet: +`/dev/root` 229 GiB used / 62 GiB available; Docker images 77.74 GiB with +56.55 GiB reclaimable and BuildKit cache 27.84 GiB with 8.579 GiB reclaimable. +Cleanup is deferred until Phase D closes so the explicit active/rollback +images remain recoverable. + +**Phase D settled-reference image gate (`PASS / PRE-ROLL`, 2026-09-02).** The +single reader build completed sequentially as `qdl-v2-python:2.0.0-a3f423d`, +image ID `sha256:dccc24b2f4a9f4f0aff58551cfe09f7ce8e7a5f42e4ef372b27b0c5b6b58d659`. +OCI labels report `revision=a3f423d`, `release=2.0.0-a3f423d`, and the image +runs as `qdl:qdl` (`10001:10001`). A disposable non-root/read-only/no-network +import of `BinanceUsdmReferenceAdapter` passed. The new operator-only override +is `/home/bobby/.local/state/qdl-v2/phase54-settled-reference-a3f423d/query-image.override.yml` +(SHA-256 `a2c804c9fbc786618a41a1e99d351cc03f91137b660ecec623000760bf0051c8`) +and changes only `query_v2_1` and `query_v2_2`; the paired rollback override is +SHA-256 `8154ece125e412e636470dd8acc79443c0f7a04ce26c435874f4a5865d738640` +and restores only those roles to `2.0.0-136c52e`. The exact current Compose +stack, stable env and all existing overrides rendered successfully with the new +file appended last. The next approved action is serial `query_v2_2`, then +`query_v2_1`, with bounded health/restart/image checks after each role. No +other service is eligible. + +**Phase D settled-reference reader handoff and paired proof (`PASS / NO-ORDER`, +2026-09-02).** `query_v2_2` then `query_v2_1` were serially recreated through +the recorded override. Both now run +`qdl-v2-python:2.0.0-a3f423d` / image ID +`sha256:dccc24b2f4a9f4f0aff58551cfe09f7ce8e7a5f42e4ef372b27b0c5b6b58d659`, +are healthy and have restart count `0`. The exact `136c52e` override/image is +retained as rollback; streams, V1, Kafka/Redis/SQLite, Rust, ingestors, bar +edge, projectors, Trading System, alpha services and order path were unchanged. + +One concurrent disposable UID-1000/no-broker/no-Gateway/Risk Binance USD-M and +OKX Swap reference pair then used V2_PRIMARY, sealed bindings, mTLS/JWT leaf +mounts and tmpfs-only cursor/audit state. Every request used `require_all=true`. +Binance `BTCUSDT` passed: FUNDING_RATE `90` observations +(`/fapi/v1/fundingRate`), OPEN_INTEREST `30` +(`/futures/data/openInterestHist`), LONG_SHORT_RATIO `30` +(`/futures/data/globalLongShortAccountRatio`), TAKER_FLOW `30` +(`/futures/data/takerlongshortRatio`), CONTINUOUS BASIS `30` +(`data.binance.vision` plus `/fapi/v1/klines`) and CONTRACT_METADATA `1` +(`/fapi/v1/exchangeInfo`). OKX `BTC-USDT-SWAP` passed the two products its +sealed binding actually declares: FUNDING_RATE `90` +(`/api/v5/public/funding-rate-history`) and CONTRACT_METADATA `1` +(`/api/v5/public/instruments`). This is real provider evidence, not generated +data or a cross-venue fallback. An initial Binance invocation stopped before a +provider request because CLI enum `continuous` was invalid; the rerun used the +declared `CONTINUOUS` enum and both sides passed. All `--rm` containers +self-removed and the scoped `/tmp` cursor/audit/output directory was removed. +No alpha durable state, V1 fallback read, Gateway/Risk request, signal/sizing +mutation, database/Redis test row or broker action was created. Phase D remains +in progress only for the approved paired representative alpha-class no-order +proofs. + +**Phase 54.2 Phase C release-render defect (`FAIL-CLOSED / SOURCE FIX`, +2026-09-02).** The bounded canonical-reader preflight stopped before any +container recreation because the newly sealed `reader-image.override.yml` was +not valid YAML: the generator emitted each service name without its required +mapping colon. This was a control-plane renderer defect, not a provider, +reader, runtime or data-plane failure. The release bundle remains unconsumed; +V1, Kafka topology/offsets, Redis, SQLite, Rust, ingestors, bar edge, +projectors, Trading System, alpha and every order path remain unchanged. The +in-scope repair is exactly one generator character plus a regression that +parses both generated candidate and per-role rollback overrides with the same +YAML parser used by Compose. The bundle must then be regenerated atomically +from the unchanged candidate image and exact inventory/compiler digests; only +after a successful canonical Compose render may the already approved four-role +serial reader/stream packet resume. Rollback remains the recorded mixed-image +per-role selector map; no broad cleanup or durable mutation is authorized. + +**Phase 54.2 Phase C release-render source gate (`PASS / SOURCE-ONLY`, +2026-09-02).** The generator now emits a valid YAML service mapping for both +candidate and exact per-role rollback overrides. The regression parses each +generated override with PyYAML and verifies exactly the four permitted reader +roles and their immutable selectors, closing the blind spot that allowed a +text-count-only test to pass. In the existing immutable reader image with the +working source mounted read-only, `--network none`, non-root UID `10001`, +read-only root and tmpfs-only scratch, `tests.test_alpha_reader_release` and +`tests.test_python310_compat` passed `7/7` in `1.887s`. No image, release +bundle, Compose service, provider, V1/V2 data plane, alpha, Trading System, +Gateway/Risk or order state was changed. The next operation is commit/push +this source correction to `dev`, rebuild one canonical image from that exact +revision, regenerate the prior invalid bundle under a new versioned release +path, and rerun canonical Compose preflight before the already authorized +four-role serial handoff. + +**Phase 54.2 Phase C canonical reader/stream handoff (`PASS / DEV +PRE-RELEASE`, 2026-09-02).** Data Layer `dev@69629668a48471c6c6871d7fae98ef7ee515d8b2` +built the one shared canonical reader/stream image +`qdl-v2-python:2.0.0-6962966` / image ID +`sha256:221aceb394b9ad55661bb6d81e0b1acad6a880ac18f75b1b44d03d9b4c0c3377`. +OCI revision and release labels equal that source/release coordinate; it runs +as `qdl:qdl` (`10001:10001`). With no source mount, `--network none`, +read-only root, non-root UID and tmpfs-only scratch, the image regression +matrix passed `64/64` with `1` expected ownership skip in `7.377s`. + +The verified, repeatable real-Compose inventory/compiler inputs remained +unchanged. A new atomically sealed, secret-free bundle at +`/home/bobby/.local/state/qdl-v2/releases/2.0.0-6962966` has `18` bindings, +manifest SHA-256 `40e244b54d8c228e8cd967e5bc2b8f9a7ab5cce7570acc45b1fb99725220abe2`, +candidate override SHA-256 +`60b865c2b538af0883d1ce2417da812247d97f47e146b59226786be817e4cbaf` and +exact mixed-image rollback override SHA-256 +`6f0f7cd50788c769698e4f600d8e968cee3e47fc12b8063dcfa6dd882d9e7e15`. +Canonical Compose rendered successfully: only `query_v2_1`, `query_v2_2`, +`stream_v2_active` and `stream_v2_passive` select the candidate; Rust, +Binance/OKX ingestors, projector and bar edge retain their prior selectors. + +The approved serial handoff recreated exactly `query_v2_2`, `query_v2_1`, +the observed standby `stream_v2_passive`, then the observed lease holder +`stream_v2_active`, always with `--no-deps`. Every role is Docker healthy on +the candidate, restart count `0`, OOM `false`; both query replicas return mTLS +`READY`; stream lease converged to exactly one `READY` (`stream_v2_passive`) +and one `STANDBY` (`stream_v2_active`). Post-roll RSS was `111.8`, `197.3`, +`109.6` and `115.3 MiB` respectively, below the unchanged `512 MiB` cgroup; +the bounded eight-minute error scan found `0` fatal/panic/OOM/traceback/ +exception/not-ready/unavailable matches for each role. No V1 service, Kafka +topology/offset, Redis, SQLite, Rust core, ingestor, bar edge, projector, +Trading System, alpha, Gateway/Risk or order path was recreated, reset or +otherwise mutated. The rejected invalid `2.0.0-a04c899` bundle was never +referenced by Compose and remains only as a trace artifact until Phase E exact +cleanup inventory; it is not a rollback coordinate. Phase C is closed. The +only next permitted work is Phase D's declared disposable paired no-order +alpha proof using this sealed bundle. + +**Phase 54.2 Phase D final-BAR policy regression (`IN PROGRESS / NARROW +SOURCE FIX`, 2026-09-02).** The first paired `adaptive_hma_cpp` V2-primary +no-order attempt authenticated correctly and reached the sealed native +Binance/OKX BAR warmup routes, then failed closed with `required data exceeds +its freshness policy`; pre/post scoped execution evidence remains zero. The +cause is local and deterministic: the config-derived alpha inventory regressed +non-1m final BAR routes to a fixed `180000` ms, and Alpha Runtime also treated +its legacy `180000` ms default as a hard cap even when a sealed binding declared +a longer interval-aware policy. This is not a provider outage, retry issue or +permission to widen a generic SLA. + +Approved in-scope repair: restore the previously certified final-BAR rule of +`max(declared_policy, one complete BAR interval + 180000 ms settlement grace)` +while preserving the explicit daily Basis policy; make an *absent* alpha local +BAR cap defer to the sealed route, while an explicitly configured smaller cap +may remain stricter and an explicitly weaker cap is rejected. Required tests: +deterministic `1m/5m/15m/30m/1h/1d` inventory values, invalid interval +fail-closed behavior, sealed-binding effective-freshness selection, and the +existing compiler/parser matrix. No Data Layer provider, V2 reader/stream +image, Kafka, Redis, SQLite, Rust, ingestor, projector, V1, Trading System, +broker, order path or alpha strategy logic is in scope. Rollback is a normal +Execution Alpha source revert plus removal of only regenerated sealed binding +artifacts; the currently running canonical reader image remains unchanged. + +**Phase 54.2 Phase D final-BAR policy source gate (`PASS / RUNTIME +UNCHANGED`, 2026-09-02).** The Alpha Runtime repair was validated in the +existing shared `execution-alpha-runtime-numba:v2-primary-83b28a6` image with +the complete Execution Alpha source and Trading System SDK mounted read-only, +`--network none`, UID/GID `1000:1000`, read-only root and tmpfs-only scratch. +`tests.test_deployment_requirements`, `tests.test_data_layer_v2_runtime`, +`tests.test_v2_no_order_probe` and `tests.test_typed_execution_intent` passed +**75/75** in `5.131s`. The source test proves the interval policy for +`1m/5m/15m/30m/1h/1d`, explicit stricter cap behavior, omitted-cap sealed +binding authority and explicit weaker-cap rejection. + +A second isolated export emitted the real 93-deployment inventory with SHA-256 +`35a18db3e0c946306d6796902428028dfd6f9e0b69d7e13db4d3834319e52762`. +The unchanged Data Layer compiler admitted `18` bindings and typed-blocked +`75`, compilation SHA-256 +`a98d66c2d7fe09968bb14a36700e00a93d436051f3e9a31925ed4ffbd84d61eb`. +The Data Layer compiler/release suite passed `10/10`; the shared Trading System +binding parser accepted all `18/18` regenerated bindings with each binding's +required local consumer and manifest identity. Observed final-BAR policies are +now Binance/OKX `15m=1080000`, multi-symbol `5m=480000`, Fib `30m=1980000`, +Grid `1h=3780000`, and the explicit Basis `1d=172800000` ms. No image build, +bundle mount, service, provider, Kafka, Redis, SQLite, V1, Trading System, +alpha process, broker or execution state changed. The caller-owned temporary +inventory/binding directory is retained only for the next sealed-bundle step; +it contains no secret or durable runtime data and must be removed in Phase E. + +**Phase 54.2 Phase D projector handoff repair (`OWNER-APPROVED / ACTIVE`, +2026-09-02).** The rerun did not fall back: both native 15m alpha reads were +correctly authenticated against their sealed V2 bindings and fail-closed with +`required data exceeds its freshness policy` before any Gateway/Risk request. +This is now traced to the durable projection handoff, not the alpha policy: +`binance_bar_edge` continues to ACK real Binance and OKX final BARs, including +the bound ETH 15m routes at `06:00`, `06:15` and `06:30` UTC, while all three +long-lived projectors repeatedly report `no active stable stream gateway +accepted canonical data`. The active stream lease is presently held by +`stream_v2_passive`; mTLS health from a projector reaches both stream roles, +so the safe repair is to refresh only the projector HTTP clients and preserve +the authoritative Kafka group/checkpoints and shared durable cache. + +**Approved bounded packet.** Recreate one at a time exactly +`projector_v2`, `projector_v2_2` and `projector_v2_3` into the existing +canonical reader image `qdl-v2-python:2.0.0-6962966` +(`sha256:221aceb394b9ad55661bb6d81e0b1acad6a880ac18f75b1b44d03d9b4c0c3377`) +with the exact existing runtime/TLS/state mounts and consumer group +`stable-projector-v1`. This is a reconnect/image-normalisation repair only: +do **not** reset Kafka offsets or topology, flush Redis, delete SQLite, change +V1/Rust/ingestor/bar-edge/query/stream, recreate Trading System or alpha, or +touch any order path. Rollback is the exact three-role image +`qdl-v2-python:2.0.0-phase533-projector-631d694` +(`sha256:288e617c7ec137dfcac5e94a5977772f2e0f39eac3ff48585980c0b26b0071bb`) +under the same mounts. Normal canonical/cache catch-up writes are expected; +they are the intended durable projection, not alpha execution mutation. + +**Exit tests.** Require all three projectors running without OOM/restart and +without the rejected-canonical loop; verify a fresh V2 latest final 15m BAR +for native Binance `ETHUSDT` and OKX `ETH-USDT-SWAP`, then rerun the same +concurrent directional no-order pair. Before/after scoped PostgreSQL and +Redis checks must remain zero for the temporary alpha/Gateway namespace. Stop +and rollback the three projectors if the canonical handoff still rejects or a +cache/identity mismatch appears; do not weaken the sealed freshness policy. + +**Phase 54.2 Phase D projector image-completeness correction (`OWNER-AUTHORISED / ACTIVE`, 2026-09-02).** The bounded projector reconnect exposed a packaging defect in the candidate Python image, not a data or policy defect: the running `qdl-v2-python:2.0.0-6962966` image starts the projector health process but `python -c 'import confluent_kafka'` fails, while the retained approved rollback projector image imports `confluent_kafka==2.15.0`. A projector without its Kafka client cannot consume `md.canonical.v2`; its HTTP health process alone is therefore not projection evidence. Rust core and bar edge continue to emit/ACK real canonical/final-BAR data, but query correctly fails closed until a Kafka-capable projector materialises it. + +**Exact correction and invariants.** Rebuild the same committed Data Layer source revision `dev@6962966` without build cache as one canonical non-phase artifact `qdl-v2-python:2.0.0-6962966-r2`; source, Dockerfile, lockfile, release schema and topology are unchanged. Before rollout, prove OCI revision/release labels, non-root/read-only import of `confluent_kafka` and the stable projector modules. Then rolling-recreate exactly `projector_v2`, `projector_v2_2` and `projector_v2_3`, one at a time, to this image with their existing runtime/TLS/state mounts and group `stable-projector-v1`. Query/stream remain on their already verified reader image in this bounded repair. Do not reset Kafka offsets/topology, flush Redis, delete SQLite, recreate Rust/ingestor/bar-edge/query/stream/V1, alter Trading System/alpha or touch the order path. Rollback remains the named Kafka-capable `qdl-v2-python:2.0.0-phase533-projector-631d694` image with the same mounts. + +**Exit and cleanup.** Require all three projectors `restart=0`, `OOM=false`, no rejected-canonical loop, and a new final 15m native BAR in the V2 query cache for Binance `ETHUSDT` and OKX `ETH-USDT-SWAP`; only then rerun the existing disposable paired adaptive no-order proof. The rejected incomplete reader image is retained only while its running query/stream roles need their documented rollback coordinate; no broad image/cache prune is authorised until Phase D proof closes. + +**Phase 54.2 Phase D historical-BAR provenance correction (`IN PROGRESS / NARROW SOURCE FIX`, 2026-09-02).** With the repaired projectors, authenticated native Binance/OKX 15m V2 reads now return 700 complete, authoritative, gap-free final bars and a LIVE tail. The paired alpha probe still correctly stops before Gateway/Risk because Alpha Runtime permits only `LAST_EVENT_STALE` on historical rows: Binance history also carries `BACKFILLED`, and OKX bars carry `FIELD_MISSING` because the native candle does not expose a trade-count field. These are declared BAR provenance/optional-field facts, not missing OHLC values, synthetic history, a stale tail, or a reason to relax execution quality. The bounded repair changes only shared Alpha Runtime historical-BAR validation: permit `BACKFILLED` and `FIELD_MISSING` alongside `LAST_EVENT_STALE` for non-current BAR warmup rows; current tail remains `LIVE`/final, and non-BAR, unknown flags, gaps, incomplete/non-authoritative rows, source-policy mismatch and stale current bars remain fail-closed. Required gates are deterministic validator regressions for Binance/OKX historical flags and rejection cases, the focused source suite, then the same disposable Binance/OKX directional proof. No Data Layer runtime, provider, Kafka, Redis, SQLite, V1, Trading System, alpha strategy/config, broker or order state is in scope. + +**Phase 54.2 Phase D V2 stream-loop ownership correction (`IN PROGRESS / NARROW SOURCE FIX`, 2026-09-02).** The provenance gate now passes, and the paired no-order probe reaches its signed stream reconnect. It then fails before Gateway/Risk because `DataLayerGateway` constructs the async V2 adapter on its dedicated `AsyncSdkRunner`, while legacy `V2PubSub` opens a second event loop for the same client. The resulting cross-loop `asyncio.Event` error is deterministic and cannot be fixed by retrying. The bounded correction is to schedule `V2PubSub` on the adapter-owner runner when it is supplied by `DataLayerGateway`; its existing standalone thread/loop remains only for directly constructed test adapters. It changes no route, freshness rule, provider call, stream protocol, consumer identity, runtime role, data-plane state or order path. Required gates: owner-loop regression, existing PubSub acknowledgement/backpressure/final-bar tests, complete focused source suite, then the same disposable paired Binance/OKX no-order proof with pre/post zero-mutation evidence. Rollback is an Execution Alpha source revert; no Data Layer runtime mutation is required for this source correction. + +**Phase 54.2 Phase D owner-loop gate and first paired class (`PASS / NO-ORDER`, 2026-09-02).** The shared runtime now runs gateway-created typed streams and their cursor acknowledgements on the adapter-owner `AsyncSdkRunner`; direct test-only PubSub use retains its standalone loop. The new owner-loop regression plus all focused source gates passed **77/77** in `4.902s` in the existing shared Numba image with complete Alpha source and Trading System SDK mounted read-only, non-root, network-disabled and tmpfs-only. The first real class pair, `adaptive_hma_cpp` 15m, then passed concurrently against native Binance USD-M `ETHUSDT` and OKX Swap `ETH-USDT-SWAP`: each read 700 FIFO closed bars, observed its initial V2 final bar, verified two signed-cursor `REPLAYING -> LIVE` handoffs, and reached one no-order Gateway/Risk preview preserving `data_layer_contract=V2_PRIMARY`. Both previews returned the expected `PREVIEW_REJECTED` rather than creating an execution action. The active stable consumer manifest revisions were read from runtime and used exactly (`9` Binance, `8` OKX); no claim was made from the stale probe revision. Scoped PostgreSQL evidence for the 20-minute proof window is zero for `execution_sessions`, `orders`, `fills` and `order_brackets`; the exact temporary Redis namespace is empty and both `--rm` containers removed themselves. Projector/query/stream/V1/Kafka/Redis/SQLite/Trading System/alpha services and order paths remain unchanged. The phase stays open only for its three already-approved representative pairs: multi-symbol, bracket and grid/L2; no new class, role or topology is authorised. + +**Phase 54.2 Phase D mark/index request-shape diagnosis (`IN PROGRESS / ALPHA SOURCE ONLY`, 2026-09-02).** The Fib bracket pair proved that active reader manifests are revision `9` (Binance) and `8` (OKX), and each explicitly admits the bound `MARK_INDEX_PRICE` identity under `crypto_liquid_v2`. The request nevertheless failed access control because the Alpha Runtime adds its generic `45s` session-liveness attribute to a mark/index reference snapshot; that attribute is deliberately absent from the registered mark entitlement. This is an Alpha request-shape defect, not a provider, manifest, Rust, projector, Kafka or entitlement gap. The bounded correction leaves all Data Layer source/runtime/topology/state untouched: Alpha stops attaching a WebSocket session SLA to `MARK_INDEX_PRICE`, retains the explicit SLA for TRADE/QUOTE/L2, and reruns the approved Fib/Grid no-order pairs. No retry-based acceptance, SLA widening, manifest mutation or runtime recreation is permitted. + +**Phase 54.2 Phase D mark/index source-plane correction (`IN PROGRESS / ALPHA SOURCE ONLY`, 2026-09-02).** After the liveness attribute was removed, the same authenticated Fib pair reached the generic V2 snapshot endpoint and exposed the actual source-plane mismatch: the bound mark/index route is a `REFERENCE` fresh snapshot, while Alpha Runtime still asks the durable stable-source catalog for a generic cache snapshot. That catalog correctly has no reference-route materialization key, so the query reader raises before Gateway/Risk. The narrow Alpha correction calls the existing signed `/v2/market-data/reference:batch` contract for one `MARK_INDEX_PRICE` result, preserves the sealed native identity/policy and `ALPHA` grade, validates exact `OK`/one-row/lineage/unit/decimal semantics, and returns the existing typed advisory shape. No Data Layer runtime, image, manifest, provider, Kafka, Redis, SQLite, V1, Rust, query/stream, Trading System, service or order path changes. This does not widen an SLA or synthesize data; quote/L2 remain their existing durable-cache routes. + +**Phase 54.2 Phase D L2 bootstrap/live repair (`IN PROGRESS / NARROW SHARED RUST CORE`, 2026-09-02).** The real disposable Dynamic Grid Binance proof reached the sealed V2 L2 route after BAR/quote/mark/index and signed cursor handoff passed, then correctly failed closed at `BOOK_SNAPSHOT/ETHUSDT`: the exact typed status was `STALE`, `LAST_EVENT_STALE`, age `9088ms`, versus the sealed `2000ms` bound. In the same read, `BOOK_DELTA` was `LIVE`, age `845ms`, session LIVE, complete and gap-free. This isolates one semantic defect: the one existing `snapshot_refresh_seconds=30` setting currently serves two different jobs, provider bootstrap/renewal and materializing a durable current book view. The provider anchor is healthy and must remain 30 seconds; using it as the public snapshot cadence makes a 2-second L2 route structurally impossible. + +**Approved in-scope correction.** Add a provider-neutral `materialized_snapshot_interval_ms` only to the Rust-core L2 mapping. It is distinct from the existing provider `snapshot_refresh_seconds`, which remains unchanged in the native ingestor configuration. Opt in only the two currently demanded Dynamic Grid ETH perpetual books (`BINANCE/USDM/ETHUSDT`, `OKX/SWAP/ETH-USDT-SWAP`) at `1000ms`; no all-universe expansion, vendor bypass, source fallback, new endpoint, service, topic, worker or per-symbol container is allowed. A verified core emits at most one materialized top-100 `BOOK_SNAPSHOT` per declared cadence after genuine delta/keepalive state progression; it preserves the native sequence/generation, partition ordering, dedup, gap/resync block and the original lossless `BOOK_DELTA`. The 30-second Binance REST anchor and OKX websocket bootstrap renewal stay unchanged. + +**Exit, rollback and decision boundary.** Source gates must prove compiler separation of provider-refresh and materialization cadence; reject malformed/out-of-range values; prove Binance and OKX verified delta materialization at 1 second, no materialization during gap/resync/duplicate and no mutation of raw-ingestor config. Then build one immutable Rust image and roll only existing `rust_core`, `rust_core_2`, `rust_core_3` with regenerated `core.json`, `core-002.json`, `core-003.json`; retain the exact current image/runtime files as rollback. No V1, Kafka topology/offsets, Redis/SQLite deletion, ingestor, query/stream, Trading System, alpha service, state or order path may change. Real acceptance is the same Dynamic Grid no-order pair on Binance/OKX: 180 final 1h bars, signed BAR reconnect, a fresh sequence-verified snapshot plus delta, LIMIT/OCO `PREVIEW_REJECTED`, zero scoped execution rows and self-removal of disposable clients. Measure added canonical event rate, Rust CPU/RAM and disk delta; any gap, OOM, duplicate or unexplained data loss fails closed and rolls the three core roles back. + +**Runtime-render guard alignment (`IN PROGRESS / SOURCE-ONLY`, 2026-09-02).** The existing bounded three-core refresh tool currently allows only the historical `snapshot_refresh_seconds` transition, so it would correctly reject this distinct field. Extend that guard, rather than introduce a new rollout script: prove each retained provider refresh remains byte-equivalent at `30s`; permit only the explicit `materialized_snapshot_interval_ms=1000` addition for `binance-usdm-ethusdt-book-primary-v2` and `okx-swap-eth-usdt-swap-book-primary-v2`; reject every other binding/member/field/image/authority mutation. Required gate: deterministic dry-run/apply rollback tests plus the compiler/deployment and full Rust core suites. This remains one three-core packet with the already recorded rollback boundary. + +**Runtime packet prepared (`OWNER-APPROVED / PENDING APPLY`, 2026-09-02).** Source `fix/phase54-l2-bootstrap@0a122a8` passed the focused deployment/render suite (`30/30`) and Rust core suite (`30 passed, 1 explicit isolated-Redis skip`). Immutable candidate `qdl-v2-rust:2.0.0-0a122a8-l2hot` is digest `sha256:c63d54f05cd00407f7440e48d10d13daf3beea581cc624cd5c748e88aaf15ee0`; its OCI revision label is `0a122a8`. The read-only actual-runtime dry run against `session-liveness-43cdbe3-20260829T162719Z` proves authority SHA `1cd55d7...981fb107` is unchanged and only `core.json` (`0cc241...c1674ab -> f77e31...0b4e26`), `core-002.json` (`d6e25f...7dc9cd -> f2b322...16c680`) and `core-003.json` (`2a7605...a47b2f -> 568089...2e4978`) will change, plus the rollout image selector. Apply writes exact previous bytes to a new private rollback directory, then rolling-recreates only `rust_core`, `rust_core_2`, `rust_core_3`, each with `--no-deps`. V1, Kafka topology/offsets, Redis/SQLite, ingestors, bar edge, projectors, query/stream, Trading System, alpha and order path remain excluded. Rollback restores the three named JSON bytes and selector, then recreates only those same three roles to retained digest `sha256:cfb686cf23fce8bea8c9c29c31630571bb6aad1b3a137f6dae1d28644649951f`. + +**Pre-roll runtime safety finding (`ACTIVE / MUST CLOSE BEFORE ACCEPTANCE`, 2026-09-02).** The bounded config apply completed and created the declared rollback receipt, but it ran in a Docker user namespace that persisted the updated private `rollout.env` as `nobody:nogroup 0600`; repair only its ownership and the new rollback directory to the normal host operator account before Compose can consume it. Separately, `rust_core` member 1 was already `OOMKilled=true`, exit `137`, four hours before this packet; members 2/3 are still live at approximately `252/275MiB` of their `512MiB` limits. No core was recreated after discovering this. The Phase D exit now requires identifying and bounding the core's live dedup/state retention before recreate, then all three members must remain `OOM=false`; a restart without that evidence is not acceptance. + +**Bounded dedup repair (`IN PROGRESS / SAME THREE-CORE PACKET`, 2026-09-02).** The active core configs reveal the direct retained-memory pressure: `dedup_capacity=1,000,000`, represented in both a `HashSet>` and FIFO `VecDeque>`, while the transactional Kafka bridge atomically commits canonical/quarantine outputs and raw offsets. It is a process-local replay guard, not durable correctness authority. Lower only the stable generated bound to `100,000`: still more than 390 times the maximum `256`-event transaction batch and a substantial reconnect window, while Kafka producer idempotence, transaction fencing, durable cursor/offset commits, canonical event IDs and projector idempotence remain unchanged. Extend the existing three-core refresh validator to allow exactly `1,000,000 -> 100,000` plus the already-declared two ETH materialization additions, reject all other core/binding changes, and test this transition deterministically. Rebuild one final immutable Rust image from that committed source; prior candidate and active `cfb686...49951f` remain rollback until post-roll memory/continuity acceptance completes. No new service, worker, topic, endpoint or data-plane reset is authorised. + +**Candidate reuse clarification (`APPROVED / SOURCE-ONLY`, 2026-09-02).** The bounded-dedup change affects only the Python runtime-config compiler and sealed JSON, not the Rust binary. Do not manufacture a third Rust image: retain already-built `sha256:c63d54...aaf15ee0` as the exact tested L2 binary and permit the refresh tool to reuse an already-selected immutable digest when only the bounded JSON changes. The tool must state whether the selector changes and count only actual writes; its rollback retains the intermediate candidate config, while the first packet's rollback retains the original `cfb686...49951f` image selector and JSON bytes for full reversal. + +**Phase 54.2-D Dynamic Grid L2 consumer proof (`PASS / CLOSED`, 2026-09-02).** No Data Layer source, topology, offset, cache, role or authority mutation was required for the final consumer proof. Against the active Rust core digest `sha256:c63d54...aaf15ee0`, an isolated V2_PRIMARY Dynamic Grid client consumed native Binance USD-M `ETHUSDT` and OKX Swap `ETH-USDT-SWAP` through their sealed bindings. Each proof read `180` final closed `1h` bars, quote and mark/index, a fresh verified top-100 L2 snapshot, and two signed reconnect handoffs ending in non-reset sequence-bearing `BOOK_DELTA`; both observed the valid `SNAPSHOT_REQUIRED` bootstrap followed by `REPLAYING` (with `LIVE` delivered on this run). The consumer reached only an isolated no-order Gateway/Risk preview, where typed LIMIT/OCO requests were terminal `PREVIEW_REJECTED`. + +Post-proof evidence: all three Rust cores were running with `restart=0` and `OOM=false`; the Dynamic Grid scoped PostgreSQL counts remained zero for `execution_sessions`, `orders`, `fills` and `order_brackets`; both `--rm` consumer clients self-removed; and the exact disposable no-order Gateway was removed. V1, Kafka, Redis, SQLite, ingestors, projectors, query/stream, Trading System, alpha services and order paths were not changed. This is real-provider V2 consumer evidence for the declared two ETH L2 bindings, not a broad alpha paper rollout or a claim about unbound symbols. + +**Phase 54.2-D cleanup evidence (`COMPLETE / SCOPED`, 2026-09-02).** After confirming no container reference, removed exactly two test-only images: `trading-system-phase4-test:bracket-context-20260901-r2` (`sha256:e0a95c...`) and `tradingsystem-image:1.2.0-09fda78` (`sha256:ac2fd3...`). No V2 Data Layer image, role, Kafka topic/offset, Redis key, SQLite state, volume, network, V1 service, Trading System or alpha service was removed. The Alpha Runtime probe image remains because a separately user-stopped alpha container still references it. No broad prune was used. Post-cleanup host evidence: `54,881,800,192` bytes free (`83%` used); Docker inventory is `80.42GB` images, `36.55GB` build cache and `2.11GB` reclaimable local volumes. A pre-cleanup byte baseline was not captured before this exact image deletion, so this record deliberately does not claim bytes freed. + +### Phase 54.3 - Final V2 Consumer Closure, Release And Hygiene + +Status: `OWNER-APPROVED / ACTIVE` (2026-09-02). + +**Goal.** Close the already-approved V2 work as an operable product rather than +as isolated proof: source convergence through `dev`, an authenticated +Trading-System V2 reader, one bounded no-order acceptance, release provenance, +and exact artifact/worktree cleanup across Data Layer, Trading System and +Execution Alpha. This is not a new data-plane feature phase. + +**Root cause and scope.** The active V2 server loads +`trading-system.paper.stable` manifest revision `7`, while the running Trading +System service mints JWTs with revision `6`; V2 correctly rejects the request +as `UNAUTHENTICATED`. The consumer binding currently carries route/release +provenance but not the manifest revision needed to make this mismatch +machine-checkable. Add that single immutable field to the binding contract and +its parser/generator tests. Trading System must validate the sealed revision +against its JWT configuration before startup. The only permitted runtime +recreate is `market_data_service`; V1, Kafka, Redis, SQLite, V2 ingest/core/ +projector/query/stream roles, Gateway/Risk/executor, alpha strategy containers, +brokers and order paths remain out of scope. + +**Required gates.** + +1. Rebase/integrate only reviewed V2 commits onto each repository's current + `origin/dev`; rerun affected Data Layer contract/compiler tests, Trading + System V2 bridge/deployment tests and Execution Alpha V2 runtime/no-order + tests. Every retained source change must be an ancestor of `dev` before a + worktree can be removed. +2. Generate a revision-7 Trading-System binding/JWT configuration from the + active consumer manifest, render the one-service override, and prove stale + revision `6` is rejected before any network/order action. Build one + immutable Trading System image only if source changes require it. +3. Serially recreate only `market_data_service`, preserving its logs, symbols, + V2 cursor/audit state and mTLS/JWT mounts. Rollback restores its exact + prior image and runtime binding; no store reset, topic/offset change or + alpha service start is allowed. +4. Run one 300-second real V2 no-order acceptance using the existing + Binance/OKX bindings: final BAR, quote/trade, mark/index, L2 snapshot/delta, + signed cursor/reconnect, typed Risk read-back and zero execution state + mutation. A health response alone is insufficient. +5. After all three repositories have pushed `dev` and the reviewed release + path reaches `main`, retain only active images plus one named rollback + image per runtime family. Remove only merged worktrees, stopped disposable + smoke containers, exact unreferenced test images and their unused BuildKit + cache. Never remove volumes, networks, source state, secrets or a + user-stopped alpha container without separate approval. + +**Exit.** Trading System reports its demanded V2 slices ready with no active +fallback for V2-primary routes; source/release/runtime provenance is recorded +in all three plans; all removable worktrees and test artifacts have an exact +inventory and cleanup receipt. Only then may alpha logic refinement resume on +the canonical `dev` checkout. + +**Phase 54.3 generation-bound binding source gate (`PASS / SOURCE-ONLY`, +2026-09-02).** The sealed consumer-route artifact now has an additive +`qdl.v2.consumer-route-binding.v2` form which binds the exact +`consumer_manifest_revision`; its V1 predecessor remains readable for +unpromoted consumers. The pure renderer emits V2 only when given the canonical +consumer manifest, validates its consumer ID and positive revision, and emits +no provider/runtime/order side effect. In the isolated, read-only, no-network +V2 Python image, `tests.test_phase115_universal_release` and +`tests.test_alpha_deployment_bindings` passed **19/19**. This includes V2 +round-trip/tamper rejection and retained V1 alpha compiler compatibility for +native Binance/OKX routes. No runtime artifact, service, Kafka/Redis/SQLite +state, V1 route, Trading System, alpha or order path changed. + +**Phase 54.3 Trading System binding artifact (`PREPARED / NO RUNTIME CUTOVER`, +2026-09-02).** Both active V2 query replicas read +`trading-system.paper.stable` revision `7`. A new binding was derived +deterministically from the active sealed V1 42-product artifact: it retains +release revision `2`, universal manifest +`1f04190793e56b1fff9a39f0a377d54a40f947b6a7d702dfde334f03593021e9`, all +42 exact product identities and V1 rollback data, and adds only +`consumer_manifest_revision=7`. Its V2 binding checksum is +`b2f6b0334a0bf2842880e7f917927d842f9ac06186c0902319c9e847915f2a64`. +Data Layer and Trading System parsers independently accept it for revision 7; +the Trading System parser rejects revision 6 before a network connection. The +new file is in a new `0700` operator directory and has not replaced the active +mount, changed a service, or mutated Kafka/Redis/SQLite/V1/alpha/order state. + +**Phase 54.3 reference receipt-timestamp correction (`OWNER-APPROVED / +SOURCE-ONLY`, 2026-09-02).** A real authenticated Trading-System V2 read +found that `ReferenceBatch._resolve_request` captures `received_at_ns` before +the bounded provider await. OKX mark/index legitimately returns a provider +timestamp a few tens of milliseconds later, which then appears impossible to +a strict consumer even though the request/identity/coverage are correct. The +root correction keeps the adapter's request-start timestamp for its documented +missing-provider-time fallback, but stamps a successful +`ReferenceBatchResult.received_at_ns` only after the provider fetch completes. +This makes the public receipt timestamp a real local completion boundary and +keeps provider observation time unmodified. Error/unavailable paths retain +their original early timestamp because no provider observation is accepted. + +**Invariant and gates.** No clock tolerance, freshness relaxation, cross-venue +substitution, fabricated observation, provider adapter behavior, contract +shape, runtime role, Kafka/Redis/SQLite state or order path is changed. Add a +deterministic clock/adapter regression proving adapter fallback receives the +request-start instant while a successful result reports the later completion +instant; retain existing bounded batch/coverage behavior. Run focused +reference batch plus V2 reference contract tests in an isolated no-network +image. Only after source gates pass may one standard V2 Python image be built +and exactly `query_v2_1` then `query_v2_2` be serially recreated with their +existing runtime/TLS/state mounts; the prior query image is the rollback. + +**Source-gate evidence (`PASS / NO RUNTIME MUTATION`, 2026-09-02).** In the +existing non-root, read-only, network-disabled V2 Python test image, +`python -m unittest tests.test_phase104_reference_batch +tests.test_phase113_reference_v2 tests.test_reference_l2_consumer_acceptance` +passed **49/49** in `6.589s`. The new deterministic regression proves the +adapter receives `100_000_000` as request-start while a successful result is +stamped at the later `200_000_000` completion instant. `git diff --check` and +`python -m py_compile qdl/reference/batch.py` pass. Repository-wide Ruff is +not an exit gate here: its isolated invocation reports pre-existing style and +unused-import findings throughout this legacy module/test file, including +lines untouched by this patch; no changed-line formatting or compile failure +was accepted. No container, image, provider request, cache, Kafka, Redis, +SQLite, V1, Trading System, alpha or order path changed during source +verification. + +**Reader repair rollout packet (`OWNER-APPROVED / PRE-BUILD`, 2026-09-02).** +Build one canonical shared Python reader image carrying the runtime source +correction `dev@1f64da7b2288db1debf2d5d83e819abcf1467204` as +`qdl-v2-python:2.0.1-1f64da7`, with OCI revision and release labels. The +following plan-only commit does not alter the runtime source label. Verify it +as non-root, read-only and network-disabled before use. Render an operator-only +image override that changes exactly `query_v2_1` and `query_v2_2`; serially +recreate those services with `--no-deps` and their existing runtime/TLS/state +mounts. Their exact current rollback is +`qdl-v2-python:2.0.0-6962966@sha256:221aceb394b9ad55661bb6d81e0b1acad6a880ac18f75b1b44d03d9b4c0c3377`. +The packet cannot recreate stream, projector, Rust core, bar-edge, either +ingestor, V1 or any Trading-System/alpha/order role; it cannot alter Kafka +topology/offsets, Redis, SQLite or runtime identity. Per-replica image, +health, restart and real MARK/INDEX read checks gate the next operation. + +**Reader repair rollout evidence (`PASS / RUNTIME READ-PLANE ONLY`, +2026-09-02).** The canonical image was built from +`dev@1f64da7b2288db1debf2d5d83e819abcf1467204` as +`qdl-v2-python:2.0.1-1f64da7`, immutable digest +`sha256:4a2b8d55116d582c6e142be81259695002461dc7757d23cb85858e1eaf35da24`. +Its OCI revision/release labels match the source coordinate; a non-root, +read-only, network-disabled import gate passed. Operator-only overrides in +`/home/bobby/.local/state/qdl-v2/phase543-reader-binding-20260902/` +changed exactly `query_v2_1` and then `query_v2_2`, each with +`--no-deps --force-recreate`; both are healthy, restart count zero and retain +the recorded `2.0.0-6962966` query image as exact rollback. No stream, +projector, Rust, bar-edge, ingestor, V1, Kafka, Redis, SQLite, Trading System, +alpha, Gateway/Risk or order role was recreated or reset. Real typed +mark/index reads through the active Trading-System facade passed for Binance +USD-M `BTCUSDT`/`ETHUSDT` and OKX Swap `BTC-USDT-SWAP`/`ETH-USDT-SWAP`; the +Gateway demanded-slice health subsequently reported `24/24 READY` with no +unhealthy slice. + +**Final C2 no-order packet (`OWNER-APPROVED / READY`, 2026-09-02).** Create +one disposable client on only the existing V2 stable-internal and +`executor_network` networks. It will use the four declared authenticated +consumer identities (`monitoring.multivenue.stable`, +`trading-system.paper.stable`, `alpha.binance.paper.stable`, +`alpha.okx.paper.stable`) against both query replicas and both stream aliases, +with the authoritative `RUST_PRIMARY` record. The current release compiler +resolves **299** exact products across the four identities: final BAR, +TRADE, QUOTE, MARK_INDEX_PRICE, BOOK_SNAPSHOT, BOOK_DELTA, funding, open +interest, long/short, taker flow, basis and contract metadata. It is a +300-second upper-bound real-provider read-only acceptance, not a synthetic +smoke. It verifies warmup, signed cursor replay/reconnect, reference lineage, +bounded V1 fallback then return for routes that explicitly allow it, and +fail-closed behavior for `BLOCKED` routes. It cannot call Gateway/Risk, +create an execution session/order/fill/bracket, mutate alpha signal/sizing or +connect directly to a venue. + +**C2 preconditions, rollback and cleanup.** Before start, derive a fresh +payload-free V1 fallback runtime binding from the current +`data_layer_service` container and frozen immutable V1 provenance; an old +container ID is not reusable evidence. The client mounts certificates only +read-only, copies the four declared identities to tmpfs, drops to non-root, +and removes its cursor directory before exit. Its sole mutable artifacts are +a `0700` evidence namespace with bounded receipt/stderr hashes and the +temporary client container, which is removed on either pass or failure. +V1, Kafka topology/offsets, Redis, SQLite, all persistent V2 roles, Trading +System, alpha containers and the order path remain unchanged. Failure leaves +V2 reader rollback available at the exact prior query digest and blocks +release closure; it does not retry by weakening freshness, identity, fallback +or source-authority rules. + +**C2 reference admission diagnosis (`FAIL-CLOSED / RUNTIME CONFIG DEFECT`, +2026-09-02).** The first disposable C2 client exited before any order/Gateway +action and retained only bounded stderr hashes. A non-sensitive diagnostic +added expected/actual identity, status and provider problem context to the +certification error; its focused reference suite remained `49/49` passing. +The exact first failed route is Binance USD-M perpetual `DOGEUSDT` +(`8aedd349-6999-5874-b0dd-34c6451c0b3a`) `BASIS`, returned as typed +`SOURCE_UNAVAILABLE`: `Binance native basis Rust admission is unavailable`. +Read-only inspection proves the query reader is correctly bound to the fixed +private URL `http://rust_core:8300`, but the serving `rust_core` has +`QDL_PROVIDER_ADMISSION_ENABLED=false` and the port refuses connections. +This is a rollout configuration omission, not a DOGE delisting, vendor +response, entitlement, symbol mapping, freshness failure or a reason to +remove the declared BASIS product. + +**Bounded Rust-admission repair packet (`OWNER-APPROVED / READY`, +2026-09-02).** Render one operator-only override that sets only +`QDL_PROVIDER_ADMISSION_ENABLED=true` for the existing `rust_core` service; +retain its current immutable Rust image, runtime/core mount, TLS, Kafka group, +Redis prefix and all other environment exactly as-is. Compose preflight must +prove only that one effective environment delta. Then recreate only +`rust_core` with `--no-deps --force-recreate`, preserving V1, both query +readers, streams, projectors, `rust_core_2`/`rust_core_3`, ingestors, bar edge, +Kafka topology/offsets, Redis contents, SQLite, Trading System, alpha and +order path. Rollback is the same one-service recreate using the exact +operator override with the flag `false`. + +The private admission service is Rust's existing bounded coordinator, not a +new public endpoint or worker. Its first real native-basis read may create a +short-lived, namespaced lease/CAS record under the existing +`qdl:stable:v2:provider-admission:v1` prefix; that is the intended minimal +coordination write and is neither a market-data cache reset nor an execution +mutation. Require post-roll endpoint liveness and one real native BASIS read +for each Binance `BTCUSDT`, `ETHUSDT`, `SOLUSDT`, `DOGEUSDT`, `BNBUSDT`, with +exact identity/lineage/coverage and no cross-venue fallback. Only then rerun +one full C2 acceptance from a fresh disposable namespace. + +**Rust image-selection correction (`FAIL-CLOSED / IMMEDIATE BOUNDED +ROLLBACK-CORRECTION`, 2026-09-02).** The first one-service recreate exposed a +separate deployment-provenance defect before any admission request: the base +stable environment still selected Rust image +`2.0.0-7b7388348615@sha256:3056cf849d4d767f19431af92b944698b4dbef15c044942831619d296f8cd156`, +while the active shared Rust-core set had previously been +`2.0.0-0a122a8-l2hot@sha256:c63d54f05cd00407f7440e48d10d13daf3beea581cc624cd5c748e88aaf15ee0`. +That older image starts the core loop but does not expose the required private +admission listener. The flag itself is correct; the effective image selector +was incomplete. Do not accept a mixed core revision. + +The immediate correction adds the exact existing `0a122a8-l2hot` immutable +selector to the same one-service operator override and recreates only +`rust_core` again. It is a rollback-correction to the prior active core image, +not a Rust build, topology change or additional role. The false-flag rollback +retains this same image selector. Exit requires all three core replicas to +report the identical Rust image digest, `rust_core` to listen on the private +port, restart/OOM zero, and no data-plane or execution side effect before any +native BASIS request/C2 retry. + +**Rust-admission repair evidence (`PASS / ONE-ROLE RUNTIME CORRECTION`, +2026-09-02).** The first recreate was immediately corrected before any native +BASIS request. All three core replicas now run the identical retained image +`qdl-v2-rust:2.0.0-0a122a8-l2hot@sha256:c63d54f05cd00407f7440e48d10d13daf3beea581cc624cd5c748e88aaf15ee0`, +each `running`, restart `0`, `OOMKilled=false`. The corrected `rust_core` +reports `QDL_PROVIDER_ADMISSION_ENABLED=true` and its log records +`qdl_provider_admission_started` with one sealed lane; a no-auth private GET +now returns `404` rather than connection refused, which proves listener +reachability without invoking a state-changing admission operation. V1, +query/stream/projector/ingestor/bar-edge, Kafka topology/offsets, Redis, +SQLite, Trading System, alpha and every order path remain untouched. The next +operation is one fresh full C2 namespace; its declared real BASIS products +cover Binance BTC/ETH/SOL/DOGE/BNB and will be the first actual admission/data +proof. + +**Phase 54.3 C2 full-route result (`FAIL-CLOSED / RUNTIME-CONVERGENCE DEFECT`, +2026-09-02).** The fresh 299-product, four-identity C2 progressed beyond the +previous Binance DOGE native `BASIS` failure after the bounded Rust admission +repair. The next exact failed product was `trading-system.paper.stable` on +OKX Swap perpetual `DOGE-USDT-SWAP` `BOOK_DELTA`, rejected by the existing +freshness contract. This is not a vendor substitution, disabled freshness +gate, missing C2 identity, order action or alpha mutation: the sealed product +is valid but has not been materialized by the shared realtime plane. + +**Phase 54.3 shared-runtime convergence (`APPROVED / IN PROGRESS`, +2026-09-02).** Read-only compiler comparison found that the mounted +`RUST_PRIMARY` bundle is an older partial runtime: every three core files +contain `16` bindings while the current canonical catalog/acquisition compiler +renders `182`; Binance USD-M ingestor contains `4` of `19` bindings and OKX +Swap ingestor contains `6` of `89`. The missing declared set includes the +five-liquid Binance/OKX TRADE/QUOTE/final-BAR plane plus the shared native +top-100 L2 snapshot/delta mappings for BTC, ETH, SOL, DOGE and BNB, and the +currently catalog-resolved BTC/ETH quarterly legs. This is the root cause of +the DOGE L2 C2 failure. It is a runtime materialization/provenance defect, not +a reason to relax freshness, fabricate a book, bypass Rust, add a per-symbol +worker or declare V2 certified prematurely. + +**Approved bounded repair.** Use the existing canonical +`StableSourceCatalog` and `StableAcquisitionPlan` compiler to render one +authority-preserving convergence bundle. Before any write, prove exact +identity/membership/order, authority/raw-topic/transactional identity, +provider endpoints, and all retained DNSE/V1-compatible bindings; only the +declared additions, catalog/config lineage and the previously approved bounded +dedup setting may differ. The operation may atomically replace only the three +mounted `core*.json` files and the two existing Binance/OKX ingestor JSON +files, retaining byte-for-byte backups in a new private operator directory. +It may rolling-recreate only `ingestor_binance_usdm`, `ingestor_okx_swap`, +`rust_core`, `rust_core_2`, and `rust_core_3`, using their already active +immutable images and existing mounts. It must not change authority mode, +consumer routes, TLS, Kafka topology or offsets, Redis/SQLite data, V1, +bar-edge, projectors, query/stream, Trading System, alpha containers or any +order path; normal provider-originated market-data writes are expected. + +**Required gates and rollback.** Source tests must cover the legacy partial +bundle -> canonical convergence, rejection of any non-declared field drift, +and atomic rollback. A read-only rendered diff must enumerate all five +perpetual identities and the six SOL/DOGE/BNB L2 additions without cross-venue +mixing. Runtime acceptance requires all five shared roles healthy with no +OOM/restart, real V2 top-100 `BOOK_SNAPSHOT` plus sequence-bearing +`BOOK_DELTA` for the ten Binance/OKX perpetual books, then exactly one fresh +299-product C2 no-order acceptance. Failure restores the five exact backed-up +JSON files and recreates only the same roles; V1 remains the route rollback. +No release/consumer promotion is allowed before that receipt passes. + +**Convergence source gate (`PASS / SOURCE-ONLY`, 2026-09-02).** Added the +reusable `scripts/converge_v2_primary_runtime.py` operator tool instead of a +manual file copy. It renders from the canonical compiler with the active +authority bytes, permits only the measured catalog/instrument lineage updates, +the already-approved `1,000,000 -> 100,000` core dedup bound, and the declared +session-liveness metadata. It rejects an unknown binding, retained semantic +drift, non-core/non-binding drift, unexpected dedup change and an output path +outside the private QDL state root. It stages all five exact rollback files +before atomic replacement and restores already-written files on failure. + +In a non-root, read-only, network-disabled image, +`tests.test_converge_v2_primary_runtime`, +`tests.test_refresh_v2_l2_core_runtime`, +`tests.test_refresh_v2_native_ingestor_runtime`, +`tests.test_phase104_reference_batch` and +`tests.test_reference_l2_consumer_acceptance` passed **54/54** in `17.941s`. +`py_compile` and `git diff --check` pass. No provider/runtime/data-plane +operation occurred during this gate. + +**Live convergence dry-run (`PASS / NO RUNTIME MUTATION`, 2026-09-02).** The +isolated renderer validates the active authority digest +`1cd55d7...981fb107` unchanged and reports exactly five changed mounted files: +each core is `16 -> 182` bindings with the ten Binance/OKX five-liquid +perpetual L2 source IDs included and `dedup_capacity 1,000,000 -> 100,000`; +Binance ingestor is `4 -> 19` bindings (nine native books including BTC/ETH +quarterlies) and OKX ingestor is `6 -> 89` bindings (nine native books and +its cataloged final-BAR set). Existing retained bindings have only validated +catalog/instrument lineage updates. This proves the full compiler output is +compatible with the mounted legacy subset; it has not written any runtime +file, restarted a role, called a provider or changed V1/Kafka/Redis/SQLite, +Trading System, alpha or order state. + +**Native-ingestor authority log correction (`SOURCE PASS / IMMUTABLE IMAGE AND +BOUNDED INGESTOR ROLLOUT PENDING`, 2026-09-02).** The first Binance ingestor recreate started with its rendered +authority record correctly set to `RUST_PRIMARY`, and `RawPublisher` applies +that record to the fenced sink to select `PrimaryRaw`. Read-only source +inspection found its startup JSON nevertheless writes the literal +`"RUST_SHADOW"`. This does not alter the selected sink/data behavior, but it +would make operational evidence lie about the active authority. Correct the +shared Rust startup formatter to report `config.authority.mode`, add a focused +mode regression, then build one immutable Rust image and recreate the same two +existing ingestors once each. No core, V1, Kafka/Redis/SQLite, query/stream, +projector, Trading System, alpha or order role is in this correction. + +The formatter now maps the already-fenced `AuthorityMode` enum directly to +`RUST_SHADOW`, `RUST_CANARY` or `RUST_PRIMARY`; it does not introduce a second +authority decision. The full native-ingestor Rust suite passed **13/13** in the +repository's dependency-complete, disposable builder image with network +disabled, including the new three-mode startup-log regression. The builder is +test-only. The next bounded action is to build one immutable shared Rust image, +pin it only for `ingestor_binance_usdm` and `ingestor_okx_swap`, then recreate +those two roles serially and verify their rendered authority, binding count, +restart/OOM state and real V2 L2 materialization before the one final C2 run. + +**Authority-log image (`BUILT / TWO-INGESTOR ROLLOUT PENDING`, 2026-09-02).** +Commit `7d9d5d4` built one shared, immutable Rust image +`qdl-v2-rust:2.0.1-7d9d5d4@sha256:f29fe2a6e4acd5fcff1a94601a837cfcc99b47ae5b970061fd1101155f54c334`. +Its OCI labels bind revision `7d9d5d4` and release `2.0.1-7d9d5d4`; it is the +only new runtime image required by this correction. The pre-roll override pins +only the two native ingestors to it. Their exact rollback is the currently +running immutable image +`sha256:cfb686cf23fce8bea8c9c29c31630571bb6aad1b3a137f6dae1d28644649951f`. +The Rust cores remain pinned to their existing c63 image and are excluded from +this rollout. The disposable builder tag is retained only until this bounded +evidence is complete, then removed under the phase cleanup rule. + +**C2 reference-cache freshness repair (`IN PROGRESS / PYTHON QUERY SOURCE +ONLY`, 2026-09-02).** The first post-convergence C2 client exited before its +observation window at the Binance BTC execution `MARK_INDEX_PRICE` request: +the typed V2 response was correctly fail-closed as `DATA_STALE` against the +declared `2,000ms` bound. A separate read-only query using the same Trading +System identity then returned the same official `/fapi/v1/premiumIndex` +product as `OK`, with provider observation age `1,094ms` and receipt age +`720ms`; it did not contact a venue directly, write an order or mutate a +service. The discrepancy is not an SLA reason to retry: `ReferenceBatch` +caches any snapshot for two seconds without checking the requesting +requirement's freshness limit. A source observation already around one second +old can therefore be returned from cache after its `2,000ms` execution limit. + +The in-scope correction makes the query service bypass/refetch an `OK` cached +result only when its newest provider observation no longer satisfies the +current request's declared freshness. It preserves the strict provider +timestamp, does not alter the `2,000ms` SLA, cache key, provider adapter, +source lineage or external API. The refetch is exactly once and only after a +cache-hit `DATA_STALE`; a provider result genuinely stale remains terminal +`DATA_STALE`. + +The focused valid-cache, stale-cache-refresh and genuinely-stale regression +passed **3/3**. The broader non-root, read-only, network-disabled matrix of +reference, runtime convergence, L2, native-ingestor and handoff behavior +passed **78/78** in `19.000s`; no provider or runtime role was accessed. +After source commit, build one immutable Python query image, recreate only +`query_v2_1` and `query_v2_2` serially, retain their current +`sha256:4a2b8d55116d582c6e142be81259695002461dc7757d23cb85858e1eaf35da24` +rollback image, then run exactly one fresh C2 client. No stream, core, +ingestor, projector, V1, Kafka/Redis/SQLite, Trading System, alpha or order +role is in this repair. + +**Reader-cache repair runtime packet (`OWNER-APPROVED / PRE-BUILD`, 2026-09-02).** +Build the shared canonical reader from committed source `dev@40b7165` as +`qdl-v2-python:2.0.1-40b7165`, attest its OCI revision/release labels, and +prove the focused cache-regression inside the immutable image under a +non-root, read-only, network-disabled container. Render one operator-only +query image override, then rolling-recreate exactly `query_v2_1` followed by +`query_v2_2`; require each replacement to be healthy with `restart=0` and +`OOMKilled=false` before continuing. The exact rollback coordinate is the +currently active `qdl-v2-python:2.0.1-1f64da7` +`sha256:4a2b8d55116d582c6e142be81259695002461dc7757d23cb85858e1eaf35da24` +with the same runtime mounts. Keep stream/core/ingestor/projector/V1, +Kafka offsets/topology, Redis, SQLite, Trading System, alpha and the order +path unchanged. On a healthy pair, create exactly one fresh 299-product, +four-identity, 300-second C2 no-order client namespace; it must use public +V2 query/stream only, establish signed cursor replay/reconnect, exercise its +declared V1 fallback drill, and prove zero venue-direct connections, zero +Gateway/Risk/order action and zero scoped execution rows. Failure stops here +and restores only the two query roles to the named rollback image. + +**Reader-cache repair rollout evidence (`PASS / TWO-QUERY ROLLING`, 2026-09-02).** +Built `qdl-v2-python:2.0.1-40b7165` +`sha256:0272638902ce33f45715c13c0c0bdb54975d36b1cc8bb3aee2a6b0835c0d64fe` +from `dev@40b7165`; OCI labels report the same revision and release. Its +three exact cache regressions passed in an isolated non-root, read-only, +network-disabled container (`3/3`). Compose preflight resolved that image +only for `query_v2_1` and `query_v2_2`. The roles were recreated serially; +both now report the identical digest, `healthy`, `restart=0`, `OOM=false` and +bounded clean Uvicorn startup. Core, ingestor, stream, projector, V1, +Kafka/Redis/SQLite, Trading System, alpha and order paths were unchanged. +The named `1f64da7` query image remains the exact rollback coordinate. The +only remaining acceptance for this correction is one fresh C2 namespace. + +**C2 batch-receipt freshness diagnosis (`FAIL-CLOSED / NARROW QUERY REPAIR`, +2026-09-02).** The fresh C2 client exited before its observation window on +`OKX.SWAP.PERPETUAL.ETH-USDT` execution `MARK_INDEX_PRICE`. A separately +authenticated, V2-only, read-only query using the same Trading-System identity +returned the identical official OKX mark/index route as `OK` with provider +observation age `342ms` and receipt age `227ms`; it used no V1 or direct venue +connection. The source is therefore healthy. The defect is internal batch +timing: a concurrent reference batch can receive a current snapshot early, +then complete other bounded provider work after that snapshot has crossed its +`2,000ms` execution bound. The response is correctly rejected, but the prior +repair only refreshed a literal cache hit and not an otherwise valid result +whose local receipt has aged while shared/batched work was completing. + +The bounded correction is provider-neutral and remains fail-closed: after all +initial bounded work returns, revalidate every on-demand snapshot and refresh +the identical request exactly once only when its provider observation was +within bound at `received_at_ns` but aged while cache/shared/batch work was +completing. A provider observation that is already stale when freshly received +is not refreshed and remains terminal `DATA_STALE`. It changes no SLA, +adapter, provider endpoint, manifest, cache key, topology or consumer route. +Required source gates cover fresh cached, aged shared/batch receipt, genuinely +stale fresh provider, identity preservation and no duplicate provider work; +then rebuild one query image, roll only the two query roles back through the +same exact rollback coordinate if unhealthy, and run one fresh C2 namespace. + +**Batch-receipt source gate (`PASS / SOURCE-ONLY`, 2026-09-02).** The shared +query service now classifies a snapshot as internally ageable only when its +provider observation was within the declared bound at `received_at_ns`; after +the initial bounded batch completes, an aged cache hit, leader result or +coalesced result receives one exact-identity bypass-cache refresh immediately +before response assembly. A response whose provider observation was already +stale at receipt cannot take that path and remains `DATA_STALE`. The focused +reference suite passed **14/14**: valid cache, ordinary cache refresh, +snapshot aging only after initial batch completion, shared refresh singleflight +and freshly received genuinely stale provider values. The broader non-root, +read-only, network-disabled matrix passed **81/81** in `19.083s`, covering the +reference, L2, runtime-convergence, native-ingestor and handoff boundaries. +No runtime, provider, Kafka, Redis, SQLite, V1, Trading System, alpha or order +state was accessed by these source gates. Next: commit, build one final reader +image, serially replace only the two query replicas and rerun the one C2 +namespace with the existing exact rollback image retained. + +**Assembly-freshness runtime packet (`OWNER-APPROVED / PRE-BUILD`, +2026-09-02).** Commit `e634b95` is the final narrow query correction: it moves +the one governed refresh from worker completion to response assembly, so a +snapshot cannot become stale while sibling work completes unnoticed. Build +`qdl-v2-python:2.0.1-e634b95`, attest source labels and execute the 14 focused +regressions from the immutable image. Render one operator-only selector and +recreate only `query_v2_1`, then `query_v2_2`, with health/restart/OOM checks +between them. The active `30b0d91` reader and the retained `1f64da7` reader +are rollback coordinates until the fresh C2 receipt passes. No other role, +state store, consumer, topology or order path is in scope. + +**C2 L2 quiet-session contract correction (`APPROVED / SOURCE-ONLY`, +2026-09-02).** The post-`e634b95` C2 failure is now precisely classified. Both +V2 query replicas agree that all five OKX perpetual top-100 `BOOK_SNAPSHOT` +products are verified, complete, gap-free and within their declared 60-second +snapshot bound; `BTC` `BOOK_DELTA` is actively changing, while the quieter +`ETH`/`SOL`/`DOGE`/`BNB` deltas legitimately have no provider book mutation for +longer than the generic 2-second event age. The existing Rust ingestor writes +real per-connection session evidence every second and the shared V2 state +contains a LIVE `okx-swap` public session. The defect is the consumer contract: +it declares `BOOK_DELTA` as a price snapshot with `BLOCK`/2-second recency and +omits the already-supported session-liveness predicate. This is neither a +missing binding, a provider outage, a reason to fabricate a delta nor a reason +to weaken L2 sequence/gap protection. + +**Approved narrow behavior.** For every declared Binance USD-M and OKX Swap +perpetual `BOOK_DELTA` demand, preserve the 2-second last-delta age as an +observable `LAST_EVENT_STALE` fact, but request `OBSERVE` event recency plus a +bounded real provider-session liveness check. A quiet delta is readable only +when its exact socket session is LIVE, within the declared heartbeat bound, +complete and gap-free; disconnected/unknown/stale sessions, unverified books, +duplicate/out-of-order/gap transitions and every `BLOCK` request remain +fail-closed. `BOOK_DELTA` remains a sequence/replay input, never the sole +price-selection primitive: limit/conditional risk read-back continues to use a +fresh verified `BOOK_SNAPSHOT` (and quote/mark policy where applicable). + +**Scope, gates and rollback.** Change only the shared demand/manifest policy, +the V2 quality/admission predicate and C2 validation for quiet connected book +deltas. Add deterministic regressions for LIVE quiet delta, disconnected +delta, stale session, gap/unverified rejection, and manifest round-trip for +both venues/five symbols. Run the focused contract/L2/consumer test matrix in +an isolated no-network container. No Rust provider code, image, role, Kafka +topic/offset, Redis/SQLite state, V1, Trading System, alpha, Gateway/Risk or +order action is in this source slice. If any gate fails, revert the source +slice; runtime remains on the already-active `e634b95` query image. A separate +recorded bounded bundle/query-stream handoff is required before the one final +299-product C2 retry. + +**Quiet-session source correction (`PASS / SOURCE-ONLY`, 2026-09-02).** The +three paper manifests now govern every Binance USD-M and OKX Swap perpetual +`BOOK_DELTA` as `event_recency_policy: OBSERVE` with a 45-second explicit +provider-session SLA, while retaining the existing 2-second event-age, +`BLOCK` stale/gap policies, verified sequence and full-coverage requirements. +Their governed revisions advance exactly once (`trading-system: 8`, +`alpha-binance: 10`, `alpha-okx: 9`); release routing revision `16` seals the +new manifest SHA-256 values. Validation now checks the exact session before +reporting an old event, so a disconnected channel cannot be misclassified as +merely quiet. It does not make a quiet delta price-eligible: snapshot/quote/ +mark remains the price-selection input. + +The first focused run exposed two fail-closed configuration defects before any +runtime operation: Trading System `BOOK_DELTA` had no session SLA, and the +release route still bound pre-change manifest hashes. Both were corrected in +source and YAML parsing plus manifest digest verification passed. The isolated +non-root, read-only, network-disabled matrix passed **52/52 in 30.899s**: +Phase-10.3/10.5 scope and receipt invariants, stable-route/release +certification, five-liquid Binance/OKX L2 quiet/disconnect/gap/sequence +regressions, and alpha deployment binding compilation. `git diff --check` +passed. No image, container, provider, Kafka, Redis, SQLite, V1, Trading +System, alpha, Gateway/Risk or order path was accessed by this source gate. +Next permitted operation: commit this source slice, build one canonical reader +image from that commit, then use the already-approved bounded query/stream +handoff and exactly one 299-product C2 no-order acceptance. + +**Quiet-session reader handoff (`OWNER-APPROVED / PRE-ROLL`, 2026-09-02).** +Source commit `09e5d03b3f9a068b37b4b8199b0b5337972344d6` built one canonical +reader/stream image `qdl-v2-python:2.0.1-09e5d03`, image ID +`sha256:9403705e86099de96c38d064cbc7a86e74c76c71cbbcebb028e9c587568b2dd4`. +OCI revision/release labels equal that source coordinate; an immutable, +non-root, read-only, network-disabled image gate passed the same **52/52** +matrix in `32.685s`. The one operator-only selector +`quiet-book-delta-09e5d03/reader-image.override.yml` (SHA-256 +`43978fc755a827dc2f2f4373564c097793931f07f9c34aee1d5287524132d1c5`) changes +exactly `query_v2_1`, `query_v2_2`, `stream_v2_active` and +`stream_v2_passive`. Its exact rollback selector (SHA-256 +`e446b4efa3ab5653fb7584f79a4cf984dbb3ec2ccb6db263eb75f0cbd60a8e32`) restores +both queries to `qdl-v2-python:2.0.1-e634b95@sha256:7288e921ba4902deb4477cebc911bd4c6cc93e39a368773def722717ba77b504` +and both streams to +`qdl-v2-python:2.0.0-6962966@sha256:221aceb394b9ad55661bb6d81e0b1acad6a880ac18f75b1b44d03d9b4c0c3377`. + +Compose validation with the exact current environment/override chain passed. +Rolling order is queries one at a time, then standby stream then lease-holder +stream after observing lease state. Between roles require health/dependencies, +restart count `0`, `OOMKilled=false` and expected immutable image. This packet +does not recreate or write V1, Kafka topology/offsets, Redis, SQLite, Rust +core, ingestors, bar-edge, projectors, Trading System, alpha or any order path; +normal reader audit writes only are expected. On any failure stop and recreate +only the failed role with its exact rollback selector. After all four are +healthy, run exactly one fresh 299-product, four-identity, 300-second C2 +no-order acceptance and remove only its disposable client namespace. + +**Quiet-session reader rolling evidence (`PASS / READY FOR ONE C2`, 2026-09-02).** +The approved selector was applied once, in the recorded order, to exactly +`query_v2_2`, `query_v2_1`, `stream_v2_active`, and +`stream_v2_passive`. All four now run +`qdl-v2-python:2.0.1-09e5d03@sha256:9403705e86099de96c38d064cbc7a86e74c76c71cbbcebb028e9c587568b2dd4`, +are Docker-healthy with restart count `0` and `OOMKilled=false`. The stream +lease converged normally: `stable-stream-active` holds epoch `15`; passive is +the expected standby. Query/stream logs contained no new error record during +the bounded post-roll check. V1, Kafka topology/offsets, Redis, SQLite, Rust +core, ingestors, bar edge, projectors, Trading System, alpha and order paths +were not recreated or mutated. The only remaining operation in this bounded +packet is one fresh 299-product, four-identity, 300-second C2 no-order +acceptance from a disposable, execution-network-only client; its V1 +provenance/binding will be derived afresh from the running V1 container and +the client namespace will be removed after its receipt is retained. + +**C2 DOGE BAR continuity diagnosis and bounded repair packet (`APPROVED / IN +PROGRESS`, 2026-09-02).** The first wrapper-only preflight stopped before the +unprivileged client could start because its public static wrapper had mode +`0700`; it made no V2 request and was retained separately as preflight +evidence. The fresh client then reached the real V2 route and correctly +failed closed at `trading-system.paper.stable` / Binance USD-M `DOGEUSDT` / +final `BAR 1m`: both query replicas reported a healthy current tail but +returned `OPEN_SEQUENCE_GAP` for the governed warmup. A read-only canonical +spool audit found one exact market-time discontinuity, 26 missing final opens +from `2026-09-02T12:15:00Z` through `2026-09-02T12:41:00Z`, between retained +opens `12:15` and `12:42`; this is real durable history loss, not an SLA, +session, reader, or source-lineage issue. + +**Repair scope and invariant.** Stop only the existing +`binance_bar_edge` container, retain a hash-recorded checkpoint backup, remove +only `binance-usdm-dogeusdt-bar-1m` from its checkpoint map, then start the +same container/image/runtime. Its established bootstrap path must fetch 1,000 +closed Binance rows, compare every open against the canonical cache and +publish only missing final opens through the normal Kafka/canonical/projector +pipeline. Expected normal data-plane writes are the 26 real missing BARs; +there is no direct SQLite write, image build, source change, new topology, +offset reset, Redis/SQLite flush/deletion, V1/Trading System/alpha/order +mutation. If bootstrap or health fails, stop only this role, restore its exact +checkpoint backup and start it on its unchanged image. After projector catch-up, +prove both replicas return a contiguous governed DOGE warmup and run one new +299-product C2 no-order client; the failed client does not count as acceptance. + +**DOGE BAR repair evidence (`PASS / READY FOR FRESH C2`, 2026-09-02).** A +read-only real-provider preflight through the same Binance bar wrapper returned +exactly 1,000 contiguous closed `DOGEUSDT 1m` rows and covered the full missing +window. The stopped role's checkpoint backup was retained, then only the DOGE +`1m` watermark was removed and byte-verified after installation. Starting the +same `qdl-v2-python:2.0.0-7c8db16@sha256:b87b03fbde11a913e9e057b17886e7d7a2d457f2f1b12b4dab4c687f3ec21ea8` +bar-edge container produced one normal bootstrap acknowledgement: 1,000 +provider rows, `973` already durable and `27` published (the 26 repaired +historical bars plus the next naturally closed BAR). The role is running with +restart `0` and `OOMKilled=false`. After projector catch-up, a read-only spool +audit reported `gap_segments=0`; authenticated Trading-System-paper warmups +through both V2 replicas returned 1,000 LIVE, complete, gap-free rows at the +same watermark `4169` and tail open. No V1, topology, offsets, Redis/SQLite +reset/delete, Rust, ingestor, query/stream role, Trading System, alpha or order +path changed. The sole next operation is one fresh 299-product, four-identity, +300-second C2 client with the same V1 rollback drill. + +**C2 quiet `BOOK_DELTA` SDK admission correction (`APPROVED / SOURCE-ONLY`, +2026-09-02).** The repaired C2 client reached the next governed route and +failed closed only at the Trading-System-paper `OKX.SWAP.PERPETUAL.DOGE-USDT` +`BOOK_DELTA` snapshot because `qdl_sdk.client._validate_query_payload` still +accepted a quiet, connected execution-grade `TRADE` but rejected the +equivalent explicitly governed `BOOK_DELTA` before the existing C2 continuity +validator could classify it. A direct authenticated typed read immediately +afterward from both V2 query replicas reported the same product `LIVE`, +complete, gap-free, session-live, within the 45-second session SLA and +`execution_eligible=false` solely because its last book mutation was quiet. + +**Narrow correction and invariant.** Generalize that SDK exception only to +`BOOK_DELTA` with `event_recency_policy=OBSERVE`, stale event recency, a LIVE +provider session, bounded declared session liveness, complete coverage and no +open gap. It remains non-executable continuity/replay input. `BOOK_SNAPSHOT`, +`QUOTE`, `MARK_INDEX_PRICE`, every `BLOCK` request, missing/expired/ +disconnected session, incomplete coverage and open sequence gap remain +fail-closed. No manifest, Rust/provider, durable store, Kafka/Redis/SQLite, +runtime role, V1, Trading System, alpha, Gateway/Risk or order behavior is in +this source slice. + +**Exit, rollback and next boundary.** Add SDK unit regressions for the allowed +quiet delta and the rejected session-SLA/session-state/gap/coverage/price-feed +variants, then run the focused SDK/C2 acceptance matrix in an isolated +no-network image. If any test fails, revert this source slice; the active +reader image remains untouched. Only after source exit may one canonical +reader/stream image be built and the already-approved four reader roles be +rolled with `09e5d03` retained as exact rollback, followed by one fresh +four-identity 300-second C2 acceptance. + +**Quiet `BOOK_DELTA` SDK source exit (`PASS / READY FOR BOUNDED READER ROLL`, +2026-09-02).** `qdl_sdk.client` now centralizes the existing quiet-continuity +exception. It preserves legacy `TRADE` behavior and admits `BOOK_DELTA` only +with the explicitly governed `OBSERVE` policy, LIVE provider session, declared +and in-bound session liveness, complete coverage and no open gap. It does not +admit `BOOK_SNAPSHOT` or any price-bearing feed, and it does not change +`execution_eligible=false`: a quiet delta is still non-executable replay +evidence. The initial regression correctly showed that a missing session SLA +is rejected even earlier at `DataRequirement` construction; the test now +asserts that stronger contract boundary. + +`git diff --check` passed. The isolated immutable-image, non-root, read-only, +network-disabled matrix passed **62/62 in 28.551s**: +`test_qdl_sdk_stream_projection`, Phase-10.3/10.5 acceptance and release +certification, five-liquid handoff, and alpha deployment bindings. It covers +allowed quiet delta, missing SLA, `BLOCK`, disconnected session, open gap, +incomplete coverage and quiet `BOOK_SNAPSHOT` rejection. No runtime role, +provider, Kafka/Redis/SQLite, V1, Trading System, alpha, Gateway/Risk or order +path was accessed. Next: commit this source-only slice; build one canonical +reader/stream image, roll only the approved two query and two stream roles, +then run exactly one fresh C2 client after normal health checks. + +**Quiet `BOOK_DELTA` reader handoff (`OWNER-APPROVED / PRE-ROLL`, 2026-09-02).** +Committed source `43faf3d20630bcc4720af326e0b60c9aeaa0e601` built exactly one +canonical shared reader/stream image, +`qdl-v2-python:2.0.1-43faf3d@sha256:6090b3a6c1c6bc431a329ab85cad7fe61750a33dbed3a5bb2d264c532f211545`. +Its OCI revision/version labels equal `43faf3d`/`2.0.1`, and its configured +container user is non-root `qdl:qdl`. The immutable no-network, read-only, +non-root image gate passed the same **62/62 in 32.349s** matrix. + +Pre-roll evidence shows precisely four affected roles, all healthy, restart +`0`, non-OOM and currently on the exact rollback image +`qdl-v2-python:2.0.1-09e5d03@sha256:9403705e86099de96c38d064cbc7a86e74c76c71cbbcebb028e9c587568b2dd4`: +`query_v2_1`, `query_v2_2`, `stream_v2_active`, `stream_v2_passive`. The +runtime selector will change only their image field, serially queries first +then current stream standby and lease holder; any failing role is recreated +only with that exact `09e5d03` selector. V1, Kafka topology/offsets, Redis, +SQLite, Rust, ingestors, bar edge, projectors, Trading System, alpha and order +paths remain excluded. Pre-roll disk capacity is `32G` free on `/`. + +**Fresh C2 packet after quiet-delta correction (`OWNER-APPROVED / IN +PROGRESS`, 2026-09-02).** Run exactly one disposable four-identity, 300-second +Phase-10.5 C2 client from the candidate image on `executor_network`. It can +reach only `https://query_v2_1:8200`, `https://query_v2_2:8200`, +`qdl-v2-stream-a:8210`, `qdl-v2-stream-b:8210` and the existing local V1 +fallback endpoint for the manifest-authorized local comparison. It has a +read-only root filesystem, tmpfs-only cursor/input state, dropped privileges +after narrowly copying declared mTLS/JWT files, no Docker socket, no provider +credential, no Trading System/Gateway/Risk credential and no order path. + +The serving V1 container was read-only verified unchanged against its frozen +`v1.2.4` provenance: image +`sha256:dbfb57844977513ae7ec0a4782e04da0213028a789753c6b991f26043b615d65`, +container-ID hash `0ec6292fc6fd94f88410cff826674e17ba8076fe63f5933f4258263c3217bd7a`. +The fresh evidence namespace therefore seals the same valid V1 binding rather +than mutating/recreating V1. C2 must prove all declared products across +monitoring, Trading-System paper, Binance paper alpha and OKX paper alpha: +warmup, signed cursor/replay/reconnect, V2-primary selection, permitted +`V2 -> V1 -> V2` fallback and `BLOCKED` no-fallback behavior. Exit requires +`order_actions=0`, `provider_connections=0`, temporary cursor removal and no +unexpected runtime mutation. Any failure retains only bounded diagnostic +evidence and leaves V1 plus the exact `09e5d03` reader rollback selector +available. + +**C2 bootstrap preflight correction (`NO ENDPOINT REQUEST / CLEANED`, +2026-09-02).** Two disposable namespaces stopped at bootstrap with `setpriv: +setgroups failed: Operation not permitted`; neither made a V2/V1 request. The +first hypothesis about Docker-level `no-new-privileges` was disproved by the +second run without that flag. The actual cause is the candidate image's +intentional default `qdl` user: the bootstrap needs a one-time root process to +copy the declared files into tmpfs before it drops to UID/GID `10001`. The real +C2 command therefore uses `--user 0:0` only for that reviewed bootstrap. +`setpriv --clear-groups`, empty inheritable/ambient capabilities and +`--no-new-privs` still apply to the actual unprivileged client process. No +provider, broker/order path or runtime role was contacted or mutated; each +container was `--rm` and each wrapper-only namespace is disposable and removed +before the one real C2 probe. + +**C2 runtime-record preflight (`NO ENDPOINT REQUEST / CLEANED`, 2026-09-02).** +The first correctly dropped-privilege client reached its local command and +proved UID `10001`, no effective/inheritable/ambient capabilities and +`NoNewPrivs=1`, but exited before any V2/V1 request because the disposable +container was missing the existing read-only `/runtime/authority.json` mount. +This is a harness mount omission, not an authority or data-plane failure. The +retry mounts precisely the same already-serving runtime directory used by the +query roles (`phase103-shared-primary-e0bedff-retry.../runtime`) read-only; +it does not write or recreate it. The failed namespace contains only wrapper, +bounded error and non-secret provenance/binding files and is removed before +the real C2 probe. + +**C2 real route finding (`FAIL-CLOSED / BOUNDED BAR AUDIT REQUIRED`, +2026-09-02).** The first actual unprivileged C2 client correctly reached V2 +and stopped at `trading-system.paper.stable` / `OKX.SWAP.PERPETUAL.SOL-USDT` / +final `BAR 1m` with `required feed has an unresolved sequence gap`. Its +security receipt remains correct (UID `10001`, no effective/inheritable/ +ambient capabilities, `NoNewPrivs=1`). This is an actual durable final-BAR +continuity failure, not quiet L2 admission, fallback, provider-session SLA or +an execution action. The failed receipt is retained as bounded diagnosis; +there were no order actions, provider connections or runtime writes from the +client. + +**Next correction scope.** Before any further C2 attempt, run a read-only +audit over the full governed Binance/OKX final-BAR set to identify every open +sequence gap at once. For each confirmed gap, verify the same exact closed +window from the real venue provider, back up the relevant existing bar-edge +checkpoint, remove only the affected binding watermark(s), then restart only +the relevant existing shared bar-edge role(s) on their unchanged image. Normal +provider-to-Kafka/canonical/projector repair writes are allowed; direct SQLite +writes, synthetic bars, topology/offset reset, Redis flush, V1, Rust, +query/stream, Trading System, alpha and order mutations remain prohibited. +If the audit is clean after repair, run exactly one fresh C2 client. + +**Full governed BAR audit and repair packet (`OWNER-APPROVED / PRE-APPLY`, +2026-09-02).** A bounded read-only audit inspected the latest 1,000 distinct +opens for every `51` materialized primary final-BAR partition. It found exactly +eight gaps: `okx-swap-sol-usdt-swap-bar-1m`, +`okx-swap-eth-usdt-swap-bar-30m`, `binance-usdm-btcusdt-bar-30m`, +`binance-usdm-btcusdt-bar-5m`, `binance-usdm-bnbusdt-bar-15m`, +`binance-usdm-ethusdt-bar-15m`, `binance-usdm-ethusdt-bar-1m` and +`binance-usdm-ethusdt-bar-5m`. The same existing multiplexed +`binance_bar_edge` owns bounded history bootstrap for both Binance and OKX; +there is no per-symbol worker or new topology. + +Real-provider read-only preflight passed for every one of those eight bindings: +each Binance/OKX wrapper returned exactly `1,000` contiguous confirmed closed +rows with `test_provenance=false`. The apply packet backs up and hashes only +`phase54-alpha-demand-5edbc8c.json`, stops only `binance_bar_edge`, removes +only those eight keys from `last_open_ms` with an atomic structured JSON +rewrite, then starts its unchanged image/runtime. Its normal bootstrap must +deduplicate existing cache rows and publish real missing final BARs through the +existing Kafka/canonical/projector route. Rollback restores the exact backup +and starts only this role. No direct SQLite write, source/image change, V1, +Kafka offset/topology, Redis, Rust, ingestor, query/stream, Trading System, +alpha or order action is allowed. + +**BAR checkpoint ownership correction (`IN PROGRESS / BOUNDED RUNTIME +REPAIR`, 2026-09-02).** The atomic eight-watermark JSON rewrite correctly +preserved content but was performed by the host privileged helper, leaving the +single checkpoint `phase54-alpha-demand-5edbc8c.json` as `root:root 0600`. +The existing bar-edge deliberately runs as UID/GID `10001`; its next startup +therefore failed closed with `stable BAR checkpoint is unreadable`. This is a +local file-ownership defect introduced by the repair procedure, not a provider +or data-quality failure. No direct durable-store mutation happened after the +role stopped, and the exact pre-apply checkpoint remains retained. + +**Approved corrective scope, invariants and rollback.** Change only that +checkpoint's owner/group to `10001:10001` and its mode to `0640`, matching the +runtime directory's non-root ownership while keeping it non-world-readable; +then start only the same existing `binance_bar_edge` role on its unchanged +image/runtime. Verify restart count, OOM state and normal real-provider +bootstrap acknowledgement before re-auditing continuity. If the role cannot +read or bootstrap after the permission repair, stop only this role, restore the +captured pre-apply checkpoint byte-for-byte with UID/GID `10001:10001` and +mode `0640`, then start only this role. V1, Kafka topology/offsets, Redis, +SQLite, Rust, ingestors, projectors, query/stream, Trading System, alpha and +order paths remain excluded. A fresh C2 run remains forbidden until the +post-repair governed BAR audit is clean. + +**Shared all-interval BAR convergence (`IN PROGRESS / BOUNDED RUNTIME +HANDOFF`, 2026-09-02).** The durable route contract now exposes `299` V2 +products, including `140` unique Binance USD-M/OKX Swap BAR partitions across +the five-liquid universe and native intervals. The already-running shared +query/stream image `qdl-v2-python:2.0.1-43faf3d` parses the committed catalog +as `142` enabled crypto BAR bindings (`70` Binance, `70` OKX, plus the two +preserved legacy BTC `1m` identities). The only mismatch is operational: +the existing `binance_bar_edge` remains on its older `7c8db16` image and its +checkpoint consequently declares only `35` bindings. This is why a broad +cache audit sees unmaterialized long/native BAR lanes even though the source +and consumer manifest are already complete. + +**Approved narrow handoff.** Reuse the existing immutable +`43faf3d` Python image; do not build another image, add a service, worker, +topic, volume, group, or per-symbol process. Before replacing only the shared +`binance_bar_edge` role, perform an isolated no-network constructor test with +the exact current checkpoint expanded to the candidate's enabled binding ID +set. It must preserve every existing watermark, preserve authority/catalog/ +acquisition/cache identity, set only the `binding_ids` list to the exact +candidate set, and prove the edge marks the `107` newly enabled bindings for +normal real-provider bootstrap. Then atomically rewrite only this checkpoint, +with an exact SHA-256 backup record and non-root `10001:10001`/`0640` metadata, +and recreate only `binance_bar_edge` on `43faf3d`. The normal bootstrap is +allowed to write only authentic closed BARs through the existing +provider -> Kafka -> Rust canonical -> projector pipeline. It must not write +SQLite directly or synthesize data. + +**Exit, rollback and C2 boundary.** Require the focused final-BAR/checkpoint +regression matrix, role health/no OOM, an acknowledgement showing all `142` +bindings checkpointed, and a governed C2 BAR audit over the exact manifest +partitions before one fresh 299-product/four-identity C2 no-order acceptance. +Rollback is exact: stop only `binance_bar_edge`, restore the checkpoint backup +with the preserved non-root mode/ownership, recreate only that role on +`qdl-v2-python:2.0.0-7c8db16@sha256:b87b03fbde11a913e9e057b17886e7d7a2d457f2f1b12b4dab4c687f3ec21ea8`. +V1, Kafka topology/offsets, Redis, SQLite, Rust, ingestors, projectors, +query/stream, Trading System, alpha and order paths remain excluded. + +**Runtime-mount provenance correction (`IN PROGRESS / NO DATA-PLANE WRITE`, +2026-09-02).** The first all-interval bar-edge recreate failed before provider +bootstrap because the generic bundle env resolved `QDL_STABLE_RUNTIME_DIR` to +the query runtime (`phase103...`) rather than the role's serving runtime +(`session-liveness-43cdbe3...`). The new edge intentionally reads its sealed +`phase54-alpha-demand-5edbc8c/catalog.yaml` and `acquisition.yaml` from the +latter; the generic mount lacks those files, so startup failed closed with +`FileNotFoundError`. The checkpoint stayed at `35/140` watermarks/bindings and +the role emitted zero bootstrap acknowledgements: no provider, Kafka, +canonical, projector, Redis or SQLite data-plane write occurred. + +**Correction.** Preserve the same four compose files and the already-created +canonical image selector, but supply only the bar-edge's verified existing +runtime-directory value for Compose interpolation. Validate the rendered +service's three mounts before recreate; it must mount +`session-liveness-43cdbe3.../runtime -> /runtime:ro`, the existing state +volume and TLS volume, with no other diff. Then recreate only +`binance_bar_edge` and continue the exact 140-binding bootstrap. The rollback +selector/path remains unchanged. + +**Sealed BAR projection convergence (`IN PROGRESS / CONFIG-ONLY REPAIR`, +2026-09-02).** With the correct mount restored, the new image correctly proved +that the mounted `phase54-alpha-demand-5edbc8c` projection itself still +contains only `35` BAR bindings. The full committed/image catalog is revision +`8` and acquisition revision `16` as the checkpoint expects, but its sealed +projection was never materialized from the all-interval crypto catalog. This +is a stale control-plane projection, not a Python/Rust/provider defect. The +first corrected recreate therefore failed closed before any provider call, +with checkpoint watermarks unchanged (`35/140`). + +**Bounded materialization and rollback.** While only `binance_bar_edge` is +stopped, atomically replace just the mounted phase54 `catalog.yaml`, +`acquisition.yaml` and their projection receipt from the already tested, +immutable-image-matching committed source documents. The receipt must record +the exact `140` enabled Binance/OKX crypto BAR IDs, source SHA-256 values and +the four C2 consumer identities; DNSE remains absent. Rename the existing +three-file projection directory to a timestamped rollback directory on the +same filesystem, retaining its hashes. This changes no source checkout, +authority, identity, image, Kafka/Redis/SQLite state or consumer route. Then +recreate only the existing shared bar-edge with the verified mount/image. The +previous projection directory plus `7c8db16` image/checkpoint backup are the +complete rollback coordinate. + +**All-interval bootstrap result and final active-gap repair (`IN PROGRESS`, +2026-09-02).** The corrected shared edge started on +`qdl-v2-python:2.0.1-43faf3d`, restored the preserved `35` watermarks and +completed real-provider bootstrap for all `140` Binance/OKX crypto BAR +bindings, publishing `87,435` authentic closed rows. Its checkpoint is now +`140/140`; all three projectors are running with restart/OOM `0`. A read-only +audit over the exact C2 route set reports `140/140` partitions present, +`0` missing, `0` insufficient, and one remaining real discontinuity: +Binance USD-M BTCUSDT `1m` on its governed preserved +`binance-usdm-btcusdt-bar-stable-001` partition. This identity is active C2 +input, so it cannot be dismissed as legacy. + +**Final bounded repair.** Verify the latest 1,000 closed BTCUSDT `1m` rows +from the real Binance provider through the existing bar wrapper, then stop +only `binance_bar_edge`, back up/hash the current 140-binding checkpoint, +remove only `binance-usdm-btcusdt-bar-1m` from `last_open_ms`, and start the +same role/image/mount. Its normal deduplicating bootstrap must fill only +missing authentic final opens; it must retain the other `139` watermarks. If +provider/bootstrap/health fails, restore this exact checkpoint and start only +this role. Re-run the exact 140-partition read-only audit; C2 remains blocked +until it reports zero missing/gapped/insufficient partitions. + +**All-interval BAR repair exit (`PASS / FRESH C2 AUTHORIZED`, 2026-09-02).** +The targeted BTCUSDT `1m` provider preflight returned `1,000` real, +contiguous, closed rows with `test_provenance=false`. Removing only its +watermark made the shared edge publish `27` missing opens while deduplicating +the other `973`; it completed the same `140`-binding bootstrap and restored a +`140/140` checkpoint. After projector catch-up, the exact governed C2 audit +reports `140/140` partitions present, `0` missing, `0` gapped, `0` +insufficient, with `156..1000` retained distinct final rows per partition. +All three projectors and the edge are running with restart/OOM `0`. The two +runtime preflight failures before this result made zero provider/Kafka writes; +the successful repairs used only normal authentic provider BAR writes. + +**Fresh C2 scope.** Run exactly one disposable `299`-product, four-identity, +`300`-second no-order acceptance from `qdl-v2-python:2.0.1-43faf3d`. The +launcher starts as root only to copy declared mTLS/JWT files into tmpfs, then +uses `setpriv` to run the actual client as UID/GID `10001` with all effective, +inheritable and ambient capabilities cleared and `NoNewPrivs=1`. It has a +read-only root filesystem, no Docker socket, provider credential, order, +Gateway/Risk or broker connection; it may reach only the two V2 query replicas, +the two V2 stream endpoints and the manifest-authorized local V1 fallback for +the required `V2 -> V1 -> V2` drill. Exit requires full route coverage, +signed cursor/reconnect, V2-primary, policy-correct fallback, zero order +actions/provider connections, and deletion of temporary cursor state. Any +failure preserves bounded evidence only and leaves V1 plus the exact bar-edge +and reader rollback coordinates available. + +**C2 launcher preflight (`NO ENDPOINT REQUEST / CLEANED`, 2026-09-02).** The +first disposable launcher stopped before the dropped-privilege client started: +the mechanically copied `run-c2.sh` wrapper had mode `0700`, so UID `10001` +could not open it. The only output was local shell `Permission denied`; no +V2/V1 endpoint, stream, provider, cursor, order or runtime state was touched. +Correct only this non-secret static wrapper to be readable/executable by the +runtime UID, remove its two local error receipts from the same disposable +namespace, then run the one real C2 probe. This launcher-only preflight does +not count as an acceptance attempt. + +**C2 BAR freshness diagnosis (`APPROVED / READ-ONLY`, 2026-09-02).** The +all-interval durable continuity gate is clean (`140/140`, no missing or open +sequence gap), but the first real C2 client subsequently stopped at a governed +OKX final `BAR 1m` freshness decision. Before any retry, run a compact, +read-only typed-status matrix for BTC, ETH, SOL, DOGE and BNB across both V2 +query replicas. The isolated client uses only the existing Trading-System +paper identity and reads state, freshness, session liveness, gap and +completeness; it has no provider, order, Gateway/Risk or write capability. +The decision boundary is narrow: repair only a confirmed shared +projection/provider-lineage defect, never relax the SLA or manufacture a bar. + +**C2 OKX final-BAR root cause and narrow repair (`APPROVED / IN PROGRESS`, +2026-09-02).** The typed read-only matrix reached both V2 query replicas with +zero provider connections/order actions. All five Binance USD-M `BAR 1m` +routes were `LIVE`, complete and gap-free at about `39-40s`; all five OKX Swap +routes were consistently complete/gap-free but `STALE` at about `1,600s`. +The shared edge logs prove the 140-route historical bootstrap succeeded, then +acknowledges only Binance recurring final BARs. The active `phase54` +acquisition projection declares all 70 OKX final BAR routes `RUST_NATIVE`, +while the running OKX Rust ingestor has only its declared realtime/L2 bindings +and does not materialize those candle routes. This is a shared ownership +mismatch, not DOGE-specific provider behaviour, a gap, or a reason to relax +freshness. + +The already-certified bound-bar compiler deliberately projects an enabled, +final OKX Swap BAR from broad `RUST_NATIVE` capability to the shared Python +REST finality edge. Repair only the stale union-projection path so a retained +multi-consumer projection can be rematerialized from its exact 140 binding IDs; +strict-load the candidate, prove all 70 Binance and 70 OKX BAR routes have +`PYTHON_REST` ownership, atomically swap only the three phase54 projection +files with an exact rollback directory, then recreate only `binance_bar_edge` +on its existing immutable image. Normal authentic provider BAR writes are +allowed. V1, Kafka topology/offsets, Redis, SQLite, Rust/ingestors/projectors, +query/stream, Trading System, alpha and order paths remain excluded. C2 may +retry only after the five-symbol/two-replica matrix is LIVE and the 140-route +continuity audit remains clean. + +**Union-projection compiler source exit (`PASS / CONFIG-ONLY REPAIR READY`, +2026-09-02).** `scripts/phase12_materialize_bound_bar_edge.py` now accepts +both its original single-consumer receipt and its own multi-consumer +`consumer_ids` receipt when retaining a declared baseline. The retained route +identity remains the exact final BAR binding IDs plus catalog/acquisition +hashes; it does not broaden a consumer, venue, symbol or interval. The +compiler preserves the legacy singleton receipt shape and emits an explicit +plural identity only for a union receipt. Its new regression proves a retained +OKX final BAR is rematerialized as `PYTHON_REST`, not silently left +`RUST_NATIVE`. + +An isolated, read-only, no-network source matrix passed **45/45** in +`17.304s`: union/retention identity, invalid/missing/non-final rejection, +strict catalog/acquisition loading, Binance/OKX native history pagination and +finality, checkpoint/cache-generation fail-closed behavior, bounded retries, +and fast final-BAR scheduling. No runtime role, provider, Kafka, Redis, +SQLite, V1, Trading System, alpha or order path was touched. Next is exactly +one configuration-only materialization from the active 140-route union, then +the bounded one-role bar-edge handoff already described above. + +**Receipt normalization preflight (`NO RUNTIME MUTATION`, 2026-09-02).** The +active union receipt is semantically correct but was copied with one literal +trailing `\\n` byte sequence, so the strict compiler rejects it rather than +silently accepting malformed JSON. Preserve the active receipt byte-for-byte; +create only a parsed/normalized diagnostic copy in the scoped repair evidence +directory, verify it has the same JSON object and route IDs, then use that copy +as the compiler input. The active projection is not changed by this preflight. + +**OKX final-BAR union materialization (`PASS / ONE-ROLE SWAP READY`, +2026-09-02).** The no-network compiler strictly materialized the normalized, +preserved active union into a scoped candidate projection. Its `140` catalog +and acquisition binding IDs are unique and identical: `70` Binance USD-M and +`70` OKX Swap final-BAR routes. Every candidate acquisition route is exactly +`PYTHON_REST`; there are no residual `RUST_NATIVE` BAR owners. Candidate +SHA-256 values are catalog `c8bc73879c631d9979d2bcd060b610a9eeb849edbd1df267a033a38ba694c527`, +acquisition `8d7976672b6cb67765ebfe5e6ea6147f721b6802240e569be63d925c131bba70` +and receipt `05acdcaa8683c974619e2155206ef551cb7bae08c23ae0ff3d13f61fe7f404f5`. +The invoking Trading System binding remains explicit at the receipt top level; +the retained baseline records the exact four C2 identities +`monitoring.multivenue.stable`, `trading-system.paper.stable`, +`alpha.binance.paper.stable` and `alpha.okx.paper.stable`, preserving the +already active union rather than narrowing an entitlement. This step contacted +no provider and changed no runtime/durable data. + +**Next bounded mutation.** Stop only the existing shared `binance_bar_edge`, +rename the active phase54 projection directory to a timestamped rollback +directory on the same filesystem, install this verified candidate as the active +phase54 directory with existing `bobby:bobby`/`0755` directory and `0644` file +permissions, then recreate only that edge with the already-running immutable +`qdl-v2-python:2.0.1-43faf3d@sha256:6090b3a6c1c6bc431a329ab85cad7fe61750a33dbed3a5bb2d264c532f211545` +and verified serving runtime mount. The old directory is the exact rollback; +restoring it and recreating the same one role returns the prior ownership model. +The edge may write only authentic provider-confirmed final BARs through its +existing Kafka -> Rust canonical -> projector path. No V1, Kafka +topology/offset, Redis, SQLite, Rust/ingestor/projector/query/stream, Trading +System, alpha or order path mutation is in scope. + +**OKX final-BAR runtime repair exit (`PASS / C2 RETRY AUTHORIZED`, +2026-09-02).** The exact one-role atomic swap retained the prior phase54 +projection under a timestamped rollback directory and recreated only +`binance_bar_edge` on the already active immutable `43faf3d` reader image. +The role is `running`, `restart=0`, `oom=false`; all three unchanged projectors +are also `running`, `restart=0`, `oom=false`. Its checkpoint now has exactly +`140` declared IDs and `140` watermarks (`70` Binance, `70` OKX). Bounded edge +logs prove authentic final-BAR acknowledgement for the bootstrap and the next +five-liquid Binance and OKX `1m` closes; no direct durable-store write or +synthetic market datum was used. + +The isolated typed status probe then read both V2 query replicas as the +restricted Trading System paper identity. All `20/20` observations (five +symbols x two venues x two replicas) are `LIVE`, complete and gap-free, with +final-BAR freshness about `7.5-8.4s`; all BAR provider-session fields are +correctly `NOT_APPLICABLE`. The initial no-client preflight omitted an explicit +root bootstrap user and stopped before any endpoint request; rerunning with the +documented bootstrap UID reached the dropped-privilege client successfully. +This is a launcher correction only, not a data-plane failure. The full C2 +client itself already validates every product's final-BAR warmup/order/parity +contracts across both replicas, so its fresh 299-product run is the governed +140-route continuity acceptance rather than a duplicate ad-hoc audit. + +**C2 retry scope.** Run exactly one disposable, four-identity, `300`-second +no-order acceptance from the existing `43faf3d` image. It must validate all +declared V2 products, including the 140 final-BAR set, reference reads, signed +cursor/reconnect and policy-correct V2 -> V1 -> V2 fallback. Exit requires +`PASS_V2_DATA_PLANE_ONLY`, zero provider connections and order actions, +removed temporary cursor state, and no mutation outside its bounded evidence +directory. Failure leaves V1 and the exact projection rollback directory +unchanged; it does not trigger another runtime change automatically. + +**C2 reference transport correction (`IN PROGRESS / SOURCE-ONLY`, +2026-09-02).** The first fresh C2 client reached the real V2 data plane but +stopped on a reference-batch HTTP `ReadTimeout`; its acceptance file is empty, +the bounded error receipt is retained, and it made no broker/order action. +This is not a BAR/provider freshness failure: every V2 reference requirement +is deliberately constructed with a `60s` provider deadline, while the C2 +launcher passed a generic `15s` timeout and the client derived only `30s` +(`15s + 15s` return margin). The query boundary permits the declared request +up to its bounded `90s` server deadline, so the disposable client was +incorrectly cancelling a valid in-flight reference contract. + +Correct only the acceptance client: derive its reference HTTP transport timeout +from the maximum declared reference-request deadline, plus the existing +bounded return margin (`60s -> 75s`), while preserving the `15s` timeout for +durable BAR/TRADE/QUOTE/L2 reads. Add deterministic tests for normal and +declared-long reference deadlines; do not alter reference product freshness, +provider retry policy, query runtime, image, Kafka/Redis/SQLite, V1, Rust, +ingestors, projectors, query/stream services, Trading System, alpha or order +path. The follow-up C2 client may mount only this committed source file over +its disposable test client path; it is not a runtime rollout or new image. + +**C2 reference transport source exit (`PASS / RETRY READY`, 2026-09-02).** +The acceptance client now computes reference HTTP transport time from the +largest declared `ReferenceRequirement.deadline_ms`, not the unrelated generic +durable-read timeout. In the governed scope, `60,000ms` produces the existing +bounded `75s` transport window; ordinary BAR/TRADE/QUOTE/L2 reads remain at +their existing `15s` timeout. No endpoint contract, freshness limit, retry +policy, runtime service or image changed. The focused isolated/no-network +regression `tests.test_phase105_identity_acceptance` passed **11/11**, +including `60s -> 75s`, short-deadline retention and empty-scope fail-closed +cases. An earlier combined command named a nonexistent test module after the +relevant 11 tests passed; it was corrected and is not counted as evidence. + +The next C2 client is a fresh disposable namespace and mounts this committed +acceptance source file read-only over +`/app/scripts/phase105_consumer_v2_identity_acceptance.py`. That isolates the +test-client correction without changing the reader/runtime image. It still +reaches only V2 query/stream and the policy-authorized V1 fallback, has no +provider/broker/Gateway/Risk path, and is deleted by Docker on exit. + +**Universal final-BAR freshness contract correction (`IN PROGRESS / SOURCE + +TWO-QUERY-ROLE ROLLOUT`, 2026-09-02).** The corrected C2 reached a real DOGE +`BAR 12h` request and exposed an actual V2 request-model error: +`max_freshness_ms` was globally clamped to one day. This violates the existing +sealed final-BAR route contract, which intentionally declares `12h=36h`, +`1d=3d`, `2d=6d`, `3d=9d` and `1w=21d` freshness windows. The fault is +independent of DOGE/provider data; the same public request would reject every +long interval before query execution. Current `max_session_liveness_ms` is +only `45s`, so no session policy is being broadened. + +Correct the shared V2 request/demand bound to the exact current maximum final +BAR policy (`21d = 1,814,400,000ms`) in the Python API and Rust demand-core, +with tests that accept the governed weekly horizon and reject one millisecond +above it. Regenerate/verify the public OpenAPI contract as required. This is a +backward-compatible expansion of a versioned numeric bound; it does not change +endpoint shape, identity, provider, finality, freshness values, source +authority or fallback policy. Build exactly one immutable Python reader image +after source tests pass, roll only `query_v2_1` then `query_v2_2` with the +active runtime/TLS/state mounts, verify no restart/OOM and both replicas accept +the long final-BAR request. Retain `43faf3d` as the explicit rollback image. +Do not recreate stream, bar-edge, Rust, ingestor, projector, V1, Kafka, Redis, +SQLite, Trading System, alpha or order path. Only after this bounded repair +passes may one final full C2 retry run. + +**Universal final-BAR freshness source exit (`PASS / QUERY ROLLOUT AUTHORIZED`, +2026-09-02).** The shared public V2 request bound and Rust demand validator now +admit exactly the sealed maximum final-BAR SLA (`1,814,400,000ms = 21d`) and +still reject `21d + 1ms`. The four V2 query routes (`snapshot`, `warmup`, +`history`, `feed status`) import the same Python contract constant; session +liveness remains independently capped at `86,400,000ms`, so this does not +weaken quiet-feed/session admission. `contracts/v2/openapi.snapshot.json` was +regenerated from the edited source and records the expanded `max_freshness_ms` +bound for the request model and all four query parameters. + +Focused isolated evidence: Python contract/API plus C2/fallback/release +regressions passed **54/54** in the existing reader image with a read-only +source mount and `--network none`; Rust `qdl-venue-core` passed **37/37** with +the repository lockfile. The first Rust attempt correctly exposed a missing +test-module import for the new private constant; that compile defect was fixed +before the passing run. A discarded offline attempt found no local crates.io +cache, so the locked dependency cache was populated only in a temporary +`/tmp` test directory; it is not source, runtime state, or a retained image +and must be removed during this slice's cleanup. `git diff --check` passed. + +The next bounded runtime step is now authorized by the approved scope: build +one immutable Python reader image from this committed source, rolling recreate +only `query_v2_1` and then `query_v2_2` with the existing runtime/TLS/state +mounts. Verify each is healthy/no-OOM and accepts declared long final-BAR +requirements; retain `qdl-v2-python:2.0.1-43faf3d` as the exact rollback +image. No other role, data store, consumer, provider connection, Trading +System, alpha or order path is in scope. A single fresh full C2 retry follows +only if both readers pass. + +**C2 L2 typed-status finding (`FAIL-CLOSED / SHARED-LIVENESS DIAGNOSIS`, +2026-09-02).** The bounded C2 retry reached the real V2 read plane through the +two freshly rolled query replicas and stopped without any order, signal, +provider-client, or durable-store action at `OKX.SWAP.PERPETUAL.DOGE-USDT / +BOOK_DELTA`: the client received `required data exceeds freshness policy` with +`C2 strict BOOK_DELTA retry requires a live provider session`. This is a real +quality admission failure, not a launcher, long-final-BAR, identity, fallback, +or missing-demand error. Read-only runtime inventory confirms that the single +shared OKX ingestor has `BOOK` bindings for all five governed swaps (`BTC`, +`ETH`, `SOL`, `DOGE`, `BNB`) and its normal 30-second snapshot-renewal loop is +active. The next bounded source investigation is to read typed +`BOOK_SNAPSHOT` and `BOOK_DELTA` status from both query replicas for all five +OKX swaps, then correct only the shared provider-session lineage if the status +proves the current heartbeat is not being joined to the corresponding logical +delta. No additional container, per-symbol worker, feed policy/SLA relaxation, +synthetic event, data-store mutation, V1, Trading System, alpha or order-path +change is permitted. A fresh C2 run remains prohibited until the typed status +and regression result are clean. + +**C2 L2 typed-status exit (`PASS / ONE RETRY AUTHORIZED`, 2026-09-02).** A +fresh disposable read-only probe used the restricted Trading System paper +identity against both active V2 query replicas. It observed all `20/20` +governed OKX L2 products (five swaps x `BOOK_SNAPSHOT` and `BOOK_DELTA` x two +replicas) as `LIVE`, complete and gap-free. Every `BOOK_DELTA`, including +`DOGE-USDT-SWAP`, has `provider_session_state=LIVE`, a bounded +`3-444ms` session-liveness age and a fresh `368-837ms` event; snapshots are +fresh `1.2-23.6s` under their declared `60s` SLA. The probe contacted only the +existing query replicas, reported `provider_connections=0` and +`order_actions=0`, and its Docker client self-removed. This proves the earlier +C2 observation was a transient session-generation transition that the existing +shared session-lineage design correctly exposed fail-closed, not a missing +symbol binding, cross-symbol mix, stale SLA, or provider-adapter defect. + +No source or runtime correction is required for that transient state. The +single remaining C2 retry is now authorized: a fresh four-identity, +300-second, no-order acceptance on the existing V2 reader/stream runtime. It +must still fail closed on a renewed non-LIVE/gap/incomplete state; no retry or +policy widening follows automatically if it fails. + +**C2 durable-BAR horizon correction (`IN PROGRESS / SHARED HARNESS + ONE EDGE +CONVERGENCE`, 2026-09-02).** The authorized retry reached the V2 reader with a +healthy L2 session and instead exposed a deterministic semantic mismatch at +`BINANCE.USDM.PERPETUAL.DOGE-USDT / BAR 12h`: C2 treated the consumer quota +`max_warmup_rows=10,000` as an exact required retained history horizon, so the +query correctly returned `PARTIAL_RESULT` when its bounded durable cache held +fewer than ten thousand 12-hour bars. This is not a missing BAR, a provider +failure, a stale quality admission, or a server-side partial response being +accepted. The source policy deliberately caps durable bootstrap history to +three years for long intervals because neither venue can truthfully provide +ten thousand weekly/long-duration bars; requests above retained availability +must use the existing explicit `FRESH_SNAPSHOT` history path and return typed +coverage rather than invented rows. + +The repair has two bounded parts. First, C2 will use an explicit 700-row +certification horizon for governed `BAR` products, while retaining the public +per-consumer `10,000` quota and all `2,500/5,000/10,000` source contract gates. +This is an acceptance workload size, not a reduction of caller capability or +an SDK/endpoint policy change. Second, converge the one live shared +`binance_bar_edge` from its observed old `1,000`-row environment to the +already committed canonical `10,000` bound. The edge will receive a new +namespaced checkpoint so it makes authentic, bounded provider history reads +and normal Kafka/canonical/cache writes for the existing 140 governed bindings; +no current state is erased. Its exact old image/config/checkpoint remain the +rollback. The initial retained horizon is still interval-aware and capped at +three years, so this step cannot imply fictional 10,000-week data. + +Required exit: deterministic helper/unit tests for the 700-row C2 horizon and +the unchanged `10,000` quota; a read-only two-venue 700-row warmup sample; one +role health/restart/OOM check after the edge bootstrap; then exactly one fresh +four-identity 300-second C2 acceptance. No V1, Kafka topology/offsets, +Redis/SQLite deletion or flush, Rust/ingestor/projector/query/stream role, +Trading System, alpha or order-path mutation is in scope. If provider history +cannot complete an interval-aware bounded window, the edge stays fail-closed +and the old checkpoint/image is restored; no retry silently lowers coverage. + +**C2-horizon source gate (`PASS / RUNTIME UNCHANGED`, 2026-09-02).** The C2 +client now derives a certification-only `700`-row BAR request from its sealed +product requirement, leaving the manifest's public `max_warmup_rows=10,000` +quota unchanged. Non-BAR requirements retain their exact request object. The +focused non-network, read-only, UID-`10001` matrix in the immutable +`qdl-v2-python:2.0.1-413683a` image passed `54/54` in `5.986s`, including the +new BAR-bound and non-BAR identity regressions plus the existing C2, +identity/fallback and Phase-10.5 acceptance cases. No provider, runtime role, +Kafka/Redis/SQLite state, V1, Trading System, alpha or order path changed. +The next permitted action is the already approved one-role bar-edge +convergence using a fresh checkpoint; source rollback is the committed helper +revert and runtime rollback is the recorded prior image/config/checkpoint. + +**Bar-edge history convergence packet (`APPROVED / PRE-FLIGHT PASS`, +2026-09-02).** The active service is healthy with no restart or OOM and is +currently the already certified immutable +`qdl-v2-python:2.0.1-43faf3d@sha256:6090b3a6c1c6bc431a329ab85cad7fe61750a33dbed3a5bb2d264c532f211545`. +Its sole divergence is an inherited overlay setting +`QDL_STABLE_BAR_WARMUP_ROWS=1000` and +`QDL_STABLE_BAR_MAX_CATCHUP_ROWS=1000`; canonical Compose already declares +`10000` for both. The one-role packet overlays only those two values and a +fresh state path +`/var/lib/qdl-stable/runtime/phase54-alpha-demand-5edbc8c.history10000-20260902.json`, +then recreates only `binance_bar_edge` without dependencies. It keeps the +current image, runtime/TLS mounts, Kafka topology/offsets, Redis, SQLite, V1, +Rust/ingestors/projectors/readers, Trading System, alpha and order path +unchanged. Rollback is exactly the existing image plus its old `1000` values +and `/var/lib/qdl-stable/runtime/phase54-alpha-demand-5edbc8c.json` checkpoint. +Preflight disk is `30 GB` available; no prune occurs while authentic bootstrap +writes are in flight. + +**Packet render correction (`PASS / NO RECREATE YET`, 2026-09-02).** The first +Compose render exposed an old environment-file default for +`QDL_STABLE_RUNTIME_DIR`; it would have mounted an earlier runtime bundle than +the currently serving edge. The packet was stopped before mutation and then +rendered again with the exact active read-only bind mount +`session-liveness-43cdbe3-20260829T162719Z/runtime`. The final render proves +the unchanged image, `10000/10000` history limits, `0.10s` final-bar +settlement delay, fresh checkpoint and exact active runtime/TLS/volume mounts. +This is a preflight correction, not a runtime change. + +**History-convergence execution (`FAIL-CLOSED / ROLLBACK REQUIRED`, +2026-09-02).** The one approved edge recreate started with the rendered exact +mounts, correct `10000/10000` values, restart count `0` and no OOM. It +authentically ACKed `130` existing Binance/OKX BAR bindings before the next +OKX history page was rejected by the shared validator with +`RuntimeError: OKX closed-bar history contains a time gap`. This is a real +provider-history continuity failure under the expanded retention horizon, not +a health-only failure, retry budget issue, synthetic record, or a reason to +lower/ignore the gap. The edge entered its existing fail-closed cycle while +the durable records already ACKed remained valid idempotent normal market-data +writes. The packet therefore rolls only this role back to the exact prior +`1000` overlay/state path; it does not erase the fresh checkpoint or any +acknowledged data, and it does not touch V1, Kafka topology/offsets, Redis, +SQLite, Rust, readers, Trading System, alpha or orders. Source investigation +must reproduce and repair OKX paginated historical continuity before any later +larger-retention packet is permitted. + +**Provider-gap diagnosis and corrected C2 invariant (`IN PROGRESS / +SOURCE-ONLY`, 2026-09-02).** A disposable, read-only, real-provider probe of +the exact failed binding `OKX SOL-USDT-SWAP / BAR 1m` returned `FULL` page +coverage and `10,000` confirmed rows but one authentic two-minute step at +`1787905740000 -> 1787905860000`. The provider's own historical series is +therefore not contiguous over that ten-thousand-row range. C2 must not turn a +per-consumer *ceiling* into an unconditional durable-retention promise, nor +fill or ignore that native gap. Its governed BAR requirement is corrected to +the smaller of `700`, the declared client quota and the shared three-year +interval-aware durable capacity (`1w=156`, `3d=365`, `2d=547`, then `700` for +the shorter supported intervals). Larger strategy-specific warmups remain the +existing explicit `FRESH_SNAPSHOT` provider-history path, which returns typed +coverage and stays blocked on a real gap. The shared capacity calculation will +be exported from the bar-edge module and reused by C2; no new service, +provider bypass, coverage relaxation, runtime mutation or data synthesis is +allowed. Required source exit is deterministic short/long-interval capacity +tests plus the focused no-network C2 matrix; the one final real C2 retry uses +the restored healthy `1000` checkpoint and the corrected exact horizon. + +**Interval-aware durable-capacity source gate (`PASS / RUNTIME UNCHANGED`, +2026-09-02).** `qdl.runtime.stable_bar_edge` now exposes the same +provider-neutral three-year capacity calculation that its bootstrap uses, and +the C2 harness takes the minimum of that capacity, `700` and the sealed public +quota. It preserves a row-based explicit warmup object where one was supplied. +The deterministic regression proves the public `10000` request remains +unchanged, a `12h` C2 proof remains `700`, and `1w` uses the truthful `156` +rows. The isolated immutable-image, non-network, read-only UID-`10001` matrix +passed `105/105` in `15.184s`, covering C2/identity/fallback, BAR bootstrap, +strict pagination/gap rejection and interval canonicalisation. The failed +expanded checkpoint remains preserved only as runtime evidence; no service, +provider, Kafka/Redis/SQLite state, V1, Trading System, alpha or order path +changed during this source gate. The next permitted action is one fresh +four-identity C2 receipt on the restored edge; it must still prove real +two-venue warmup rather than relying on this test alone. + +**Final C2 interval-aware packet (`APPROVED / IN PROGRESS`, 2026-09-02).** One +fresh evidence namespace pins source commit `27dd967` read-only into the +otherwise existing immutable client image +`qdl-v2-python:2.0.1-413683a@sha256:4125cb95e2954cdef8bc1a97a43b3cb82a61530328b544126d8170e2dbbcb17c`. +It mounts only the committed `qdl/` source and changed C2 helper so its shared +capacity import is exact; it does not build an image or alter a service. The +`--rm` client runs on `executor_network`, copies only the four sealed paper +identities into tmpfs, drops to UID `10001` with `NoNewPrivs` and no effective +capabilities, and observes exactly `300s`. It reaches existing query/stream +and permitted V1 local fallback endpoints only. It has no provider credentials +or Docker socket, and cannot invoke broker, Gateway, Risk, order, signal, +sizing or alpha state. Exit requires `299` product views, V2-primary quality, +bounded 700/interval-capacity BAR warmup, signed cursor/reconnect, correct +fallback policy, zero provider connections/order actions and removed cursor +state. Any failure leaves all runtime roles unchanged. + +**C2 launcher preflight (`NON-EXECUTION INPUT FIX`, 2026-09-02).** The first +disposable container exited at its shell entrypoint with `Permission denied` +opening the read-only mounted bootstrap script. It never executed bootstrap, +copied an identity, contacted V1/V2, created a cursor or wrote an evidence +receipt; Docker removed it. The two private evidence scripts were changed from +host-only mode to read-only `0644` and will be invoked through `/bin/sh`, which +does not grant the client any additional privilege. This launcher correction +does not count as a C2 run and changed no runtime role or data plane. + +**C2 privilege-drop preflight (`NON-EXECUTION LAUNCHER FIX`, 2026-09-02).** A +second disposable launch also stopped before bootstrap because the immutable +image defaults to UID `10001`, while the bootstrap intentionally needs a +brief root phase only to copy sealed identity material into tmpfs before +dropping it. A network-disabled, read-only probe proves that explicit +`--user 0:0` followed by the existing `setpriv --reuid=10001 --regid=10001 +--clear-groups --inh-caps=-all --ambient-caps=-all --no-new-privs` produces +UID/GID `10001`, no effective/permitted/inheritable/ambient capabilities and +`NoNewPrivs=1`. The actual client will use exactly that bootstrap boundary; +root has no network-only execution code, Docker socket or lasting evidence +write path. The two failed launchers self-removed before identity copy, V1/V2 +request, cursor creation or data action, so neither is an acceptance attempt. + +**BOOK_SNAPSHOT receipt correction (`IN PROGRESS / SOURCE-ONLY`, +2026-09-02).** The first real client reached V2 under the required non-root +boundary and failed at `Trading System / OKX DOGE / BOOK_SNAPSHOT` before its +300-second observation. The manifest declares a `60s` snapshot freshness SLA, +but the C2 query path incorrectly used its generic `15s` transport deadline +and only permitted retry for quote/trade/book-delta continuity feeds. A +verified book snapshot has no stream-session liveness field by design; it must +be allowed to wait through its own declared renewal cadence only while typed +status remains identity-matched, `LIVE` or `STALE`, complete and gap-free with +`NOT_APPLICABLE` session state. This does not accept stale data: the next +snapshot must still pass the original V2 freshness admission before the fixed +deadline. The correction is limited to C2's client timeout/retry policy and +adds positive plus gap/mismatched-state regressions. No data-plane service, +provider or consumer configuration changes. + +**BOOK_SNAPSHOT source gate (`PASS / READY FOR ONE FRESH C2`, 2026-09-02).** +The C2 client now derives the strict `BOOK_SNAPSHOT` request deadline from the +declared snapshot freshness SLA (`60s` for the governed execution books), not +the unrelated generic `15s` transport default. A stale snapshot is still never +accepted: retry is allowed only if the typed status matches the same instrument, +feed and source policy, remains `LIVE` or `STALE`, is complete and gap-free, +and declares `NOT_APPLICABLE` session liveness as required for snapshot +delivery. A gap, a conventional provider session field or any identity/policy +mismatch fails closed. The retry cadence is one second in production; the unit +test substitutes a millisecond delay solely to keep deterministic no-network +tests fast. + +Evidence: the existing immutable reader image +`qdl-v2-python:2.0.1-413683a@sha256:4125cb95e2954cdef8bc1a97a43b3cb82a61530328b544126d8170e2dbbcb17c` +ran the read-only, `--network none`, UID `10001` regression matrix: +`python -m unittest tests.test_phase103_consumer_receipt_harness tests.test_phase105_consumer_acceptance tests.test_phase105_identity_acceptance tests.test_phase105_fallback_acceptance tests.test_phaseb_bar_history_bootstrap tests.test_canonical_intervals`. +Result: `107/107 PASS` in `15.460s`. The first run exposed only the new unit +test's `0.25s` artificial deadline versus the real one-second retry cadence; +the test now stubs that cadence rather than changing production behavior. No +runtime role, provider request, Kafka/Redis/SQLite state, V1, Trading System, +alpha or order path changed. The one permitted next action is a fresh +299-product/four-identity/300-second no-order C2 receipt in a new namespace. + +**C2 canonical-cache capacity repair (`IN PROGRESS / SOURCE-ONLY`, 2026-09-02).** The fresh C2 client passed its non-root launcher boundary, then stopped at a real Binance USD-M `TRADE` read because the V2 cache reported `LAST_EVENT_STALE`. Read-only runtime evidence establishes the shared cause: all three projectors repeatedly receive `stable canonical ingest rejected http_status=500 detail=unavailable`; the stream-side stack identifies `BackpressureRequired: bridge max_records exhausted`. The shared `canonical-cache.sqlite3` contains exactly `1,000,000` records (`851,927,749` payload bytes, about `1.74 GiB` physical), across `178` physical partitions, with zero active checkpoints and a truthful approximately-24-hour retained horizon. The existing per-partition replay window is `10,000`, so the catalog permits `1,780,000` retained records while the static global cap permits only `1,000,000`; the cache reaches the global ceiling before its own declared partition windows/retention can coexist. This is a shared durable-capacity configuration defect, not a provider, identity, alpha, Trading System, order, or V1 fallback defect. + +Approved narrow correction: derive the stable spool record ceiling from the loaded canonical catalog's **unique physical partition keys** times the existing bounded per-partition window, with the prior one-million floor retained. The payload, physical-storage, free-disk, 24-hour retention, signed generation-bound cursor, monotonic checkpoints, idempotent event IDs and fail-closed bounds remain authoritative; the correction must not delete, compact early, flush, rebuild, or replay the live cache. Source tests must prove the old under-provisioned shape is rejected, the catalog-derived ceiling is deterministic and provider-neutral, and full/expired/active-consumer retention semantics are unchanged. Map a stream-side capacity rejection to typed temporary-unavailable only after preserving the fail-closed body and retry semantics. Runtime is explicitly excluded until source evidence passes; then one immutable Python image and a rolling recreate only of the existing seven cache users (`projector_v2`, `projector_v2_2`, `projector_v2_3`, `stream_v2_active`, `stream_v2_passive`, `query_v2_1`, `query_v2_2`) may adopt the same code/config with V1, Kafka topology/offsets, Redis, SQLite files, Rust, ingestors, Trading System, alpha and order paths untouched. Rollback is the exact active image/runtime mount and no data deletion. + +**C2 canonical-cache capacity source gate (`PASS / RUNTIME ROLLOUT READY`, 2026-09-02).** `qdl.runtime.stable` now derives one shared `StableSpoolCapacity` from the loaded source catalog before every query, stream or projector opens the cache. It counts distinct physical `partition_key` values, intentionally counting an L2 snapshot/delta pair once, retains the established `10,000` per-partition hard window and uses the higher of that product and the legacy one-million floor. The current sealed source catalog has `206` logical bindings and `188` physical partitions, so its runtime ceiling is deterministically `1,880,000` records rather than the inconsistent static `1,000,000`. Existing payload (`2 GiB`), physical storage (`3 GiB`), free-disk reserve (`512 MiB`), 24-hour age retention and cursor/checkpoint limits remain independent fail-closed bounds; this correction removes only the contradictory record ceiling. + +`/internal/v2/canonical/events` now maps only `BackpressureRequired` to a typed `503 stable canonical cache capacity temporarily unavailable`; it still rejects invalid lineage, capacity exhaustion and fenced writers, and it never converts a capacity rejection into an ACK. Focused deterministic tests cover catalog-derived capacity above the old static ceiling, unchanged legacy floor, active/expired-consumer retention, append atomicity, cursor expiry and typed `503` mapping. The immutable existing reader image `qdl-v2-python:2.0.1-413683a` ran the network-disabled, read-only UID-`10001` matrix: `184/184 PASS`, `1 skipped`, `26.486s` across C2 receipt/identity/fallback, stable edge/projector, spool, BAR bootstrap and interval contracts. A separate read-only catalog proof returned `{logical_bindings: 206, physical_partitions: 188, per_partition_records: 10000, global_max_records: 1880000}`. No provider, runtime role, Kafka/Redis/SQLite state, V1, Trading System, alpha or order path changed. + +Next permitted step is deliberately one runtime packet, not another design phase: build one immutable Python image from this commit, preserve the active reader/stream/projector images as the exact per-role rollback set, then rolling-recreate exactly the seven existing shared cache users in the order `stream_v2_passive -> stream_v2_active -> projector_v2* -> query_v2_*`. This preserves an available stream peer while the canonical sink adopts the wider bound before projectors resume catch-up. The cache files stay mounted in place; startup simply recognizes the larger bounded configuration. Verify stream ingest accepts canonical records, projectors catch up, both query replicas report fresh five-symbol Binance/OKX `TRADE`, and then run one fresh C2 299-product/four-identity/300-second no-order acceptance. V1, Rust, ingestors, Kafka topology/offsets, Redis, SQLite deletion/flush, Trading System, alpha and orders remain excluded. + +**Canonical-cache capacity runtime packet (`APPROVED / PRE-FLIGHT`, 2026-09-02).** Immutable image `qdl-v2-python:2.0.1-d9dea34@sha256:dcb15154ccff53490d152152af99a8aa808c9901c5d210135d791d74d0398bd7` is built from source commit `d9dea34`, with OCI revision/version labels matching that source. The same `184/184` network-disabled suite passed from the built image without a source mount. Runtime override and exact per-role rollback override are retained under `/home/bobby/.local/state/qdl-v2/releases/2.0.1-d9dea34/canonical-cache-capacity/`; they name only the existing seven cache users and do not create a topology, volume, network, state path or service. Pre-flight must render this override last against the active Compose stack, verify those seven image substitutions only, and preserve current mounts, TLS/runtime directory and environment. The bounded rolling packet then follows the stated stream -> projector -> query order. No cache delete/flush/replay or broader service action is authorized. + +**C2 projector memory immutable-image gate and runtime packet (`PASS / APPROVED`, +2026-09-02).** The single shared image +`qdl-v2-python:2.0.1-7352a1a@sha256:21084686cee5f6b108b34cd39183c0d839eac1723c85795f2fa50bb2b2764896` +was built from source commit `7352a1a` with matching OCI revision/version +labels. Its isolated, network-disabled, read-only UID-`10001` matrix passed +`175` tests with `1` explicit skip in `31.155s`, without a source mount. The +only approved runtime mutation is a rolling recreate of the three existing +shared projector roles: `projector_v2`, `projector_v2_2`, and `projector_v2_3`. +The final override is applied last and pins that image, `768 MiB` memory, `1024` +pending records, `16 MiB` pending bytes, `128` records per batch and `8 MiB` +per batch. The exact rollback is the current +`qdl-v2-python:2.0.1-d9dea34@sha256:dcb15154ccff53490d152152af99a8aa808c9901c5d210135d791d74d0398bd7` +with its current `512 MiB`, `2048` record and `32 MiB` pending configuration. +The packet preserves the current runtime/TLS/state mounts, Kafka group and +offsets, Kafka topology, Redis, SQLite files, V1, Rust, ingestors, query/stream, +Trading System, alpha and all order paths. It stops and rolls back immediately +on any new OOM/restart, cache admission failure, duplicate/gap evidence or +freshness regression. Runtime exit is all three projectors running without OOM, +bounded memory, accepted canonical ingestion, five-symbol Binance/OKX TRADE +freshness convergence and one fresh disposable four-identity/299-product/300s +C2 no-order receipt. No cache rebuild, flush, deletion, synthetic record or +SLA relaxation is authorized. + +**C2 projector-memory launcher correction (`NON-EXECUTION / RETRY READY`, +2026-09-02).** The first disposable C2 client proved its bootstrap boundary +(UID/GID `10001`, no effective/permitted/inheritable/ambient capabilities and +`NoNewPrivs=1`) and self-removed, then stopped before its first identity, +market-data, cursor or fallback read because the packet omitted the existing +read-only runtime bind that supplies `/runtime/authority.json`. The failure is +therefore a packet mount defect, not a provider, V2 reader, fallback or C2 +acceptance result. The retry mounts only the already active +`/home/bobby/.local/state/qdl-v2/phase103-shared-primary-e0bedff-retry-20260825T054740Z/runtime` +directory read-only at `/runtime`; it changes no service, image, Kafka/Redis/ +SQLite state, V1, Trading System, alpha or order path. Fresh evidence stays in +a new namespace, and the original failed launcher receipt remains preserved. + +**C2 resumed-quote diagnosis and acceptance-clock correction (`IN PROGRESS / SOURCE-ONLY`, 2026-09-02).** The corrected disposable read-only probe exercised the same sealed Trading System paper identity, V2 query replicas and signed gRPC stream path for `BINANCE.USDM.PERPETUAL.DOGE-USDT / QUOTE`. It found the live path healthy at the time of measurement: warmup freshness was `686ms`, the first post-warmup durable event was `587ms` old at logical offset `2288389`, and the resumed event was `662ms` old at offset `2288390`; both were source/identity matched, `LIVE`, execution-eligible and strictly increasing. This rules out a permanently stale DOGE provider/source binding and shows the earlier C2 failure occurred while the projector/cache was still converging after the authentic backlog. + +Source review nevertheless found two C2-harness correctness gaps that must be closed before a certificate is credible. First, `--observation-seconds 300` currently bounds the whole task but permits it to return early; it is not a real 300-second observation. Second, reconnect currently validates every replayed durable frame as current execution price even when the frame is correctly ordered but predates the fresh reconnect snapshot watermark. That conflates replay used only to restore consumer state with an executable current price. The narrow correction is limited to the shared C2/SDK acceptance boundary: retain strict identity, source, sequence, gap and offset checks for every replay frame; keep stale replay non-executable; require a fresh strict V2 snapshot after replay before the receipt can declare the route current/execution-ready; and hold the accepted four-consumer scope through a real bounded time window followed by full strict closing revalidation. No SLA is relaxed, no replayed payload may drive order/risk/signal/sizing, and no provider/runtime role, Kafka/Redis/SQLite state, V1, Trading System, alpha or order path changes are in scope. Required tests cover fresh live resume, valid stale replay followed by a fresh read-back, stale read-back rejection, gap/identity mismatch rejection, and the actual observation-duration floor. After source and immutable-image gates pass, one new 299-product/four-identity/300-second no-order C2 receipt is the only runtime acceptance action; its client self-removes and its scoped evidence is retained without payloads or secrets. + +**C2 projector memory/throughput repair (`APPROVED / IN PROGRESS`, 2026-09-02).** The capacity rollout exposed a separate bounded-runtime defect before C2 can be rerun. The current three existing projector roles share one Kafka group and one SQLite cache; `projector_v2_2` and `projector_v2_3` were OOM-killed at their `512 MiB` cgroup limit after authentic backlog catch-up, while `projector_v2` remains live at about `522 MiB` cgroup usage (`~385 MiB` anonymous PSS). The live passive-named stream is the active lease holder and reports `READY`; this is not a provider, lease or consumer identity failure. The single surviving projector is therefore insufficient to catch up canonical TRADE freshness promptly and C2 correctly remains fail-closed. + +Approved source scope is deliberately narrow and shared: replace the canonical HTTP sink's repeated full-JSON chunk measurement with linear bounded chunk accounting; make projector batch and pending limits explicit, bounded runtime configuration rather than a hard-coded `512` batch; bound the Confluent consumer prefetch queue; and assign only the three existing projector roles an evidence-backed `768 MiB` cgroup budget. No provider adapter, event identity, schema, Kafka topic/group/offset, Redis, SQLite data/file, V1, Rust, ingestor, query/stream, Trading System, alpha or order path changes. Source exit requires deterministic chunk-equivalence/max-byte tests, config-bound tests, existing projector/order/retention regression, full focused non-network reader matrix, and an immutable-image rerun. Runtime exit is a rolling recreate only of the three existing projector roles with the exact prior image/config retained as rollback; require all three running, no OOM/restart, accepted canonical ingestion, cache event-recency convergence across the governed five-symbol Binance/OKX TRADE set, then one fresh disposable 299-product/four-identity/300-second C2 receipt. Do not reset Kafka offsets, delete/flush the cache, synthesize records, relax freshness, or retry C2 before that evidence exists. + +**C2 projector memory source slice (`PASS / IMMUTABLE IMAGE PENDING`, 2026-09-02).** The shared HTTP sink now materializes only one bounded encoded chunk at a time. Its previous candidate-length loop repeatedly serialized the entire growing JSON payload and retained all chunks before the first request; the new exact size calculation uses the fixed UUID-envelope length plus deterministic per-item JSON length, then serializes each actual request once. Existing chunk-order, request-byte, oversized-event and acknowledgement tests cover the wire invariant. `StableRuntimeConfig` now carries projector-only record and byte batch bounds; stable projectors use `128` records, `8 MiB` per batch, `1,024` pending records and `16 MiB` pending bytes, while queries/streams preserve their prior behavior. A polled record that would exceed the byte bound is retained in-order for the next drain, never dropped or accepted above the bound. Test fixtures with a smaller pending ceiling derive a smaller default byte batch automatically. `ConfluentProjectorBroker` additionally caps its client-side queue at `16 MiB` with an `8 MiB` fetch and `2 MiB` partition fetch; this is a memory bound, not a Kafka offset or delivery-semantic change. The three existing projector services alone override the shared Python `512 MiB` ceiling to `768 MiB`; no other Python role changes. + +The first isolated test run found the new batch/pending relationship was incorrectly enforced for `query_v2`; it was narrowed to `projector_v2` before any runtime action. The final expanded immutable-base-image, network-disabled, read-only UID-`10001` regression ran C2 receipt/identity/fallback, BAR history bootstrap, interval, stable edge/projector, stable release and query readiness modules: `175 PASS`, `1 explicit skip`, `30.266s`. It includes configuration boundary, byte-deferred in-order batch drain, Kafka read-committed/checkpoint, canonical chunking/ack, spool retention and projector recovery tests. No image, service, provider, Kafka/Redis/SQLite state, V1, Trading System, alpha or order path changed during source verification. Next: build one immutable shared Python image, repeat this matrix from that image without a source mount, then use a three-projector-only rolling packet. + +**C2 resumed-quote/observation source gate (`PASS / IMMUTABLE CLIENT PENDING`, 2026-09-02).** `market_data_view_from_stream(..., replay_only=True)` now has one explicit state-recovery use: it keeps identity, source, authority, schema and gap checks fail-closed, returns old replay as `STALE` and non-execution-eligible, and never changes the default strict behavior. C2 invokes it only when the resumed logical offset is at or behind the fresh reconnect handoff watermark. It acknowledges the bounded replay state and then requires a separate strict V2 snapshot/read-back before attesting the route as current. `validate_product_view(..., state_replay=True)` likewise keeps identity/provenance/gap/payload validation while refusing to claim current quality; provider-session/freshness authority is the mandatory current read-back. A replay after the new handoff stays ordinary strict execution validation. + +The four-identity runner now performs a full opening warmup/query/cursor/reconnect/fallback sweep, remains alive through the requested observation floor, then repeats the exact governed scope at the closing boundary. Receipt evidence records requested/actual duration and opening cardinality; final revalidation has a bounded `90s` budget. This corrects the prior misleading use of `--observation-seconds` as only a task deadline. Two historical acceptance assertions were also aligned with the already-approved interval-aware C2 BAR warmup bound rather than the public `10,000` quota. + +Evidence: the network-disabled, read-only UID-`10001` matrix in existing immutable image `qdl-v2-python:2.0.1-7352a1a` passed `172/172` in `22.219s`: SDK stream projection, cursor/reconnect, C2 harness/identity/fallback, consumer acceptance, stable BAR history/canonical intervals and five-liquid handoff. New deterministic cases prove stale replay stays non-executable, gaps still fail, stale session state can be replayed only with `state_replay=True`, fresh read-back is mandatory, replay offset/watermark classification is exact, and the C2 timing helper cannot return before its declared duration. No runtime role, provider connection, Kafka/Redis/SQLite state, V1, Trading System, alpha or order path changed. + +**C2 immutable-client gate (`PASS / REAL RECEIPT IN PROGRESS`, 2026-09-02).** The correction is now packaged as `qdl-v2-python:2.0.1-0cfdc1e@sha256:ee019d6c5d78824c8fe8e21ce76c947cd1d35aa271121abe65dd844b3bf613af`, with OCI revision `0cfdc1e798a521b681f6689817e4521f5d2d52b3`, version `2.0.1-0cfdc1e` and non-root runtime user `qdl:qdl`. The same network-disabled, read-only, no-source-mount UID-`10001` matrix passed `172/172` in `22.897s`. This verifies the shipped client artifact rather than the checkout. The sole next action is one fresh four-identity `299`-product C2 no-order acceptance for a true `300s` observation; the disposable client has no Docker socket, broker/Gateway/Risk/order/signal/sizing capability or provider credential and self-removes. No service rollout is needed for this client-only correction. + +**C2 runner-mode preflight (`NON-EXECUTION INPUT FIX`, 2026-09-02).** The first new disposable launcher copied the evidence-private runner with mode `0600`; after its intentional UID-`10001` privilege drop, `/bin/sh` correctly refused to read `/run-c2.sh`. It exited before the C2 Python client, V1/V2 request, cursor, product receipt, provider connection or order-capable code ran. The preserved evidence namespace records this failed preflight. The replacement packet copies only the two public launcher scripts as read-only `0644`; sealed credentials remain copied from the read-only state mount into tmpfs only after startup. This changes no image, service, data plane, secret material, V1, Kafka/Redis/SQLite state, Trading System, alpha or order path and does not count as a C2 attempt. + +**C2 typed-snapshot boundary repair (`IN PROGRESS / SOURCE-ONLY`, 2026-09-02).** The corrected launcher reached the real C2 reader and stopped before its observation window at `monitoring.multivenue.stable / BINANCE.USDM.PERPETUAL.ETH-USDT / TRADE`. The SDK correctly returns a typed `SnapshotResponse`; the resumed-replay strict read-back passed that envelope directly to `validate_product_view`, which correctly expects the enclosed `MarketDataView`. This is an acceptance-client type-boundary defect, not a provider, symbol, freshness, replay, session, cache, V1 fallback or runtime-role defect. The narrow repair must unwrap only the typed SDK response before validation and add a regression proving the real response contract reaches the validator's view layer. No policy/SLA/coverage relaxation, source/provider change, data-plane mutation or retry counts as C2 until the immutable client gate is repeated. + +**C2 typed-snapshot first correction (`INVALIDATED / SOURCE CORRECTION REQUIRED`, 2026-09-02).** The first source correction unwrapped `SnapshotResponse.data` inside `_strict_snapshot_for_c2`, and its isolated test passed `173/173`; the immutable client also passed that matrix. The real C2 correctly exposed the omitted second caller: `_query_product_with_quality` consumes the same helper and needs the typed envelope for its own `.data` access. This is a deterministic acceptance-client interface error, not a provider, symbol, freshness, replay, session, cache, V1 fallback or runtime-role defect. The failed no-order client produced no receipt and did not reach the observation window. The correction is now narrowed further: `_strict_snapshot_for_c2` keeps its established typed `SnapshotResponse` contract for both callers; only the resumed-replay validator reads `.data`. Required regression covers the query caller preserving the envelope and the replay caller validating its inner view. No policy/SLA/coverage relaxation, source/provider change, data-plane mutation or service rollout is allowed. + +**C2 typed-snapshot final source gate (`PASS / IMMUTABLE CLIENT REBUILD REQUIRED`, 2026-09-02).** The helper again returns the public typed SDK envelope exactly as the normal query path requires; the replay-only branch alone unwraps `current.data` for `validate_product_view`. Regressions now prove both contracts: strict query retains the real `SnapshotResponse`, while state recovery validates its enclosed view after a replay and before attestation. The network-disabled, read-only, source-mounted UID-`10001` matrix passed `173/173` in `21.218s`, covering C2 query/replay callers, cursor/reconnect, identity/fallback, strict gap/identity rejection, BAR bootstrap and interval cases. No runtime role, provider, Kafka/Redis/SQLite state, V1, Trading System, alpha or order path changed. The required next step is one final immutable-client rebuild and the single approved true-300-second C2 receipt. + +**C2 typed-snapshot final immutable-client gate (`PASS / REAL RECEIPT IN PROGRESS`, 2026-09-02).** Final client `qdl-v2-python:2.0.1-6fecc6b@sha256:021186754b334ee6a926f6405da341ba24cccec13c82219b816c7ed2171212a1` was built from source commit `6fecc6b`, with matching OCI revision/version labels and `qdl:qdl` user. The complete no-source-mount, network-disabled, read-only UID-`10001` matrix passed `173/173` in `21.447s`. The only remaining C2 action is the already-approved fresh four-identity, 299-product, true-300-second no-order receipt. It has no service-recreate, provider, broker/Gateway/Risk/order/signal/sizing or alpha-state capability; V1 is read only through the governed fallback drill. + +**C2 calendar-anchor replay repair (`IN PROGRESS / SOURCE-ONLY`, 2026-09-02).** The final client reached a real governed alpha route and stopped before its observation window at `alpha.binance.paper.stable / BINANCE.USDM.PERPETUAL.DOGE-USDT / BAR 1w`. `1w` is a fixed-duration canonical interval but its documented provider grid is Monday-anchored, not Unix-epoch aligned. C2's historical replay seed performed a raw `open_time % duration` check, incorrectly rejecting a real final weekly bar that the shared provider-aware interval contract already accepts. The repair must use the shared provider-calendar alignment helper with the product venue and preserve millisecond precision, then derive prior seeds by fixed-duration subtraction. It must reject malformed nanoseconds/unanchored bars, preserve execution-BAR blocking and add Binance/OKX weekly-anchor regression. This is a C2 client calendar-contract defect only: no provider, runtime role, source data, policy/SLA, Kafka/Redis/SQLite, V1, Trading System, alpha or order path mutation is permitted; the failed client produced no certificate. + +**C2 calendar-anchor source gate (`PASS / IMMUTABLE CLIENT REBUILD REQUIRED`, 2026-09-02).** Historical BAR replay now delegates alignment to the shared `is_valid_bar_open_ms` contract with the real product venue and rejects sub-millisecond/misaligned data. The retained seed remains exactly two fixed-duration bars before the latest validated final BAR. Regression covers `1w` for Binance and OKX plus `3d` where Binance is Monday-anchored and OKX is Unix-anchored; existing execution, malformed and unbounded-history rejection stays covered. The source-mounted, network-disabled, read-only UID-`10001` matrix passed `174/174` in `23.344s`. No service, provider data, Kafka/Redis/SQLite state, V1, Trading System, alpha or order path changed. Next: build one immutable client from this source, repeat the matrix without a source mount, then run the single approved C2 receipt again. + +**C2 calendar-anchor immutable-client gate (`PASS / REAL RECEIPT IN PROGRESS`, 2026-09-02).** Immutable client `qdl-v2-python:2.0.1-95142bd@sha256:662105d7f6970093407fcda06256104c9fa9aab2ebf3c660a888622fb8aa68ad` was built from source `95142bd`, with matching OCI labels and non-root `qdl:qdl` user. The no-source-mount, network-disabled, read-only UID-`10001` matrix passed `174/174` in `24.144s`. One fresh four-identity, 299-product, true-300-second C2 no-order receipt remains the only acceptance action; it cannot alter V1, any service, Kafka/Redis/SQLite state, Trading System, alpha or order path. + +**C2 restored-cursor semantics repair (`IN PROGRESS / SOURCE-ONLY`, 2026-09-02).** The calendar-corrected client completed the strict opening read and stream checks, then stopped in the reconnect probe at `trading-system.paper.stable / OKX.SWAP.PERPETUAL.BNB-USDT / QUOTE`. A restored signed cursor is state recovery evidence, but C2 treated its first returned frame as executable when its logical offset happened to be newer than the reconnect warmup watermark. That can be a legitimately delivered older source observation from durable recovery, and the SDK correctly rejected it under the execution freshness policy. The C2 contract must treat the bounded resumed frame(s) as non-executable state replay regardless of watermark ordering, retain identity/source/gap/offset checks, then require the already-existing strict fresh V2 snapshot before attestation. The independent first live stream remains strict and execution-grade, so this neither accepts a stale price nor weakens production SDK behavior. Required regression covers stale replay on both sides of a reconnect watermark, mandatory strict read-back and gap/identity failure. No provider/runtime/cache/policy/SLA/V1/Trading System/alpha/order mutation is in scope; the failed client produced no certificate. + +**C2 restored-cursor source gate (`PASS / IMMUTABLE CLIENT REBUILD REQUIRED`, 2026-09-02).** The resumed session is now explicitly state-recovery-only: its bounded frame preserves strict identity, source, gap and monotonic-offset checks, is marked non-executable, and is followed by a required strict current V2 snapshot before C2 can attest the route. The initial live stream remains unmodified and strict. Regression proves the resumed frame remains state replay even when its logical offset is after the reconnect watermark; it also verifies historical BAR replay, execution BAR strict read-back, quiet-channel policy and gap/identity rejection. The source-mounted, network-disabled, read-only UID-`10001` matrix passed `174/174` in `22.816s`. No runtime service, provider data, Kafka/Redis/SQLite state, V1, Trading System, alpha or order path changed. Next: rebuild one immutable client, repeat that matrix without a source mount, then run C2 once. + +**C2 restored-cursor immutable-client gate (`PASS / REAL RECEIPT IN PROGRESS`, 2026-09-02).** Immutable client `qdl-v2-python:2.0.1-02f0afb@sha256:9ba060e5ab55181412490408d10daeaf2dac45815c3a4e06f58148d24141142f` was built from source `02f0afb`, with matching OCI labels and non-root `qdl:qdl` user. The no-source-mount, network-disabled, read-only UID-`10001` matrix passed `174/174` in `21.400s`. The sole remaining action is one fresh four-identity, 299-product, true-300-second C2 no-order receipt; it cannot modify any runtime role, V1, Kafka/Redis/SQLite state, Trading System, alpha or order path. + +**C2 real DOGE 1h gap diagnosis (`IN PROGRESS / READ-ONLY`, 2026-09-02).** The restored-cursor client reached the governed alpha product plane and then stopped fail-closed at `alpha.binance.paper.stable / BINANCE.USDM.PERPETUAL.DOGE-USDT / BAR 1h`: V2 warmup returned typed `required feed has an unresolved sequence gap`. This is real data-plane quality evidence, not a C2, SDK, cursor, calendar, fallback or policy defect. C2 cannot certify V2-primary while this flag exists. The next work is read-only lineage inspection of both query replicas, bar edge/projector logs and the canonical gap state for this exact binding. If repair is needed, it must be a narrow, provider-backed final-BAR resync/recovery packet that retains gap fencing until contiguous history is verified; no synthetic fill, SLA/policy relaxation, V1/Trading System/alpha/order mutation or broad cache reset is allowed. + +**C2 DOGE final-BAR targeted recovery (`APPROVED / IN PROGRESS`, 2026-09-02).** Read-only lineage proved the exact defect: canonical partition `binance-usdm-dogeusdt-bar-1h-primary-v2` has `9,002` distinct final opens with no flagged payload, but it is missing exactly `1,001` contiguous `1h` opens from `1784689200000` onward before current live rows resume. The active bar edge continues to ACK current DOGE `1h` closes, so this is a historical continuity hole retained in the cache rather than a missing binding, stale provider or C2 policy issue. Approved repair scope is one reusable, provider-neutral final-BAR history recovery primitive plus a narrowly scoped CLI for named enabled Binance/OKX bindings. It reads the bounded authenticated provider window, validates exact provider calendar continuity, reads durable opens read-only, publishes only provider-confirmed missing final opens through the existing raw Kafka/Rust canonical/projector path, and waits for cache coverage to converge. It must not clear a gap flag manually, alter edge checkpoint/watermark, reset offsets, delete/flush SQLite/Redis, recreate any service, touch V1, Trading System, alpha or order path, or use synthetic data. The DOGE invocation is fixed to `binance-usdm-dogeusdt-bar-1h` and must observe the diagnosed `1,001` missing opens before applying; a changed count fails closed. Source gates: exact binding identity, Binance/OKX calendar, duplicate/overlap, no-write dry-run, bounded publish and post-projection convergence. Runtime exit: the exact partition is contiguous for the bounded provider window, both V2 query replicas report no gap for the DOGE `1h` requirement, and one fresh C2 receipt may then proceed. Rollback is operationally fail-closed: stop the disposable repair client; already accepted authentic events remain durable evidence and are never deleted. + +**C2 targeted final-BAR recovery source gate (`PASS / IMMUTABLE REPAIR CLIENT PENDING`, 2026-09-02).** `StableBinanceBarEdge` now exposes a provider-neutral prepared repair window: it obtains bounded authenticated Binance/OKX history, validates exact opens/calendar, calculates durable coverage read-only, and can publish only still-missing final rows through the existing raw Kafka path. The apply path fences the approved missing count immediately before publish, rechecks cache generation, never advances the active edge checkpoint and has no cache/Redis/offset/service mutation path. `scripts/repair_stable_final_bar_history.py` requires named bindings, exact row window, exact `binding=count` expectation and an explicit confirmation token; dry-run performs provider/canonical validation only, while apply waits for the existing projectors to materialize the same expected opens. The script writes only temporary state under `/tmp` and uses a distinct Kafka client ID, so it cannot race the active edge checkpoint. Regression ran inside `qdl-v2-python:2.0.1-7352a1a` with read-only source mount, UID `10001`, `--network none`: `43 PASS` across durable history bootstrap, provider-calendar anchors, overlap/idempotency, cache-generation fencing, scheduled final-BAR catch-up and the new Binance/OKX targeted-repair tests (`10.796s`). `py_compile` and `git diff --check` pass. No image, service, provider connection, Kafka/Redis/SQLite data, V1, Trading System, alpha or order path changed. Next is one immutable repair-client image, then a real-provider dry-run pinned to DOGE `1h`; only if its exact count remains `1,001` may the explicit normal-data-plane repair run. + +**C2 targeted-repair bound correction (`IN PROGRESS / SOURCE-ONLY`, 2026-09-02).** The first immutable-client dry-run exited before any provider call, Kafka publish or state write because `prepare_history_repair(..., rows=10,000)` incorrectly inherited the service's ordinary `warmup_rows=500` ceiling. A normal startup warmup and a one-time provider-backed continuity repair have distinct contracts: the former remains bounded by the running service configuration; the latter must be bounded by the existing interval-aware truthful durable capacity and the global `10,000` maximum. The narrow correction changes only that repair bound, adds a regression proving a `500`-row runtime can safely prepare a larger valid repair window, then repeats the network-disabled regression and the exact DOGE dry-run. No runtime role, provider data, Kafka/Redis/SQLite state, V1, Trading System, alpha or order path changed in the failed preflight. + +**C2 targeted-repair bound source gate (`PASS / IMMUTABLE REPAIR IMAGE REBUILD REQUIRED`, 2026-09-02).** `prepare_history_repair` now distinguishes the live loop's configured startup warmup from a deliberately invoked recovery window: repair is capped only by the global `10,000` maximum and the same provider-neutral interval-aware durable capacity used for truthful retained history. The active edge configuration remains unchanged at `warmup_rows=500`; no service configuration was widened. New deterministic coverage proves an edge with a two-row normal warmup can prepare a three-row provider-confirmed recovery plan. The network-disabled, read-only, source-mounted UID-`10001` matrix passed `44/44` in `10.965s` across durable bootstrap, provider-calendar anchors, overlap/idempotency, cache-generation fencing, scheduled catch-up and targeted Binance/OKX repair. The former immutable client is superseded for repair only; no runtime role, provider call, Kafka/Redis/SQLite data, V1, Trading System, alpha or order path changed. Next: build one replacement immutable repair image, rerun the same no-source-mount test matrix, then run the exact DOGE `1h` provider dry-run with the already-approved `1,001` fence. + +**C2 DOGE repair real-provider dry-run (`PASS / APPLY TARGET RE-FENCED`, 2026-09-02).** Immutable image `qdl-v2-python:2.0.1-2b8690b` passed the no-source-mount, network-disabled, read-only UID-`10001` regression matrix `44/44` in `11.612s`. Its first provider dry-run failed closed before publishing because the previously recorded `1,001` count no longer matched the exact present cache/provider window. A follow-up read-only probe against the same authenticated Binance `10,000`-row window and canonical cache found `8,998` covered, `1,002` missing and no non-contiguous or current-final-bar ambiguity: the contiguous hole is `2026-07-22T03:00:00Z` through `2026-09-01T20:00:00Z` inclusive, while the window ends `2026-09-02T20:00:00Z`. The original fence was therefore one row short, not a reason to relax quality. Apply is re-fenced to exactly `1,002` provider-confirmed rows for the same named DOGE `1h` binding; any changed count, cache generation, provider calendar/coverage failure or non-convergence still aborts without partial workaround. This applies normal authentic BAR writes only through raw Kafka -> Rust canonical -> existing projectors/cache. It does not recreate services, advance bar-edge checkpoint/watermark, reset offsets, flush/delete state, touch V1, Trading System, alpha or order path. Rollback remains fail-closed by stopping the disposable client; already accepted authentic historical events are retained as auditable market-data evidence. + +**C2 late-backfill physical-capacity repair (`APPROVED / IN PROGRESS`, 2026-09-02).** The approved `1,002`-row authentic repair published and canonicalized, but cache convergence correctly stopped at `4` missing rows rather than pretending success. Read-only cache lineage proves why: public consumer history is bounded at `10,000` rows and the physical SQLite partition window was also exactly `10,000`; four valid older late-backfill records (`2025-07-12T23:00:00Z` through `2025-07-13T02:00:00Z` by their raw epoch values) remain in the physical replay tail, so four current provider-window rows (`2026-09-01T21:00:00Z` through `2026-09-02T00:00:00Z`) cannot coexist. This is a physical-retention defect at the late-backfill boundary, not a provider gap, Rust rejection, projector lag, duplicate, quality-policy or C2 problem. The narrow correction adds a fixed `64`-row physical late-backfill headroom per partition while retaining the public `10,000` maximum and all SDK/API quotas exactly unchanged. It is capacity-derived in the existing shared spool, not a per-symbol exception. Required tests prove public limits remain `10,000`, physical/global capacity includes bounded headroom and normal cursor/retention behavior remains bounded. Runtime packet after immutable-image validation is limited to rolling `projector_v2`, `projector_v2_2` and `projector_v2_3` with the same runtime/TLS/state mounts; it preserves Kafka offsets/topology, Redis, SQLite files/cache generation, V1, Rust/ingestors/bar-edge/query/stream, Trading System, alpha and order path. Rollback recreates only those three projectors with their current image `qdl-v2-python:2.0.1-7352a1a@sha256:21084686cee5f6b108b34cd39183c0d839eac1723c85795f2fa50bb2b2764896` and same mounts. Then a four-row provider-backed repair uses a new exact `4` fence; it may only publish the still-missing four and must converge before C2 resumes. + +**C2 late-backfill physical-capacity source gate (`PASS / IMMUTABLE IMAGE REBUILD REQUIRED`, 2026-09-02).** The shared stable spool now retains a bounded `10,064` physical-record window per partition: the public query/SDK warmup ceiling remains exactly `10,000`, and the `64` rows are explicitly reserved for late authentic backfill/revision ordering. Global cache capacity remains catalog-derived from physical partitions, so the change scales with the shared topology rather than a symbol/interval special case. The network-disabled, read-only, source-mounted UID-`10001` matrix passed `54/54` in `12.178s`, including query/projector readiness capacity, public-history contract, BAR bootstrap/repair, calendar, overlap/idempotency and final-BAR scheduling regressions. No runtime role, provider call, Kafka/Redis/SQLite data, V1, Trading System, alpha or order path changed during the source gate. Next: one immutable Python image from this commit, repeat the same matrix without a source mount, then rolling-recreate only the three projectors with their existing mounts and verify cache generation/health before the new four-row repair. + +**C2 late-backfill projector runtime preflight (`PASS / ROLLING IN PROGRESS`, 2026-09-02).** Immutable `qdl-v2-python:2.0.1-1994258@sha256:b335d609db7b55000cba7cce455a525dbc8d7793faa83b94bc2cfbaa12f64430` carries source commit `1994258` and passed the same no-source-mount, network-disabled, read-only UID-`10001` matrix `54/54` in `13.172s`. Compose render confirms the exact rollout scope resolves that digest for the existing shared Python role; the command will apply it only to `projector_v2`, `projector_v2_2` and `projector_v2_3`, one at a time, with `--no-deps --no-build --force-recreate`. The existing runtime directory remains `/home/bobby/.local/state/qdl-v2/session-liveness-43cdbe3-20260829T162719Z/runtime`, and both `stable_state`/`stable_tls` volumes retain their current identities. No additional role, network, volume, topic, offset, V1, Rust/ingestor/bar-edge/query/stream, Trading System, alpha or order-path mutation is authorized. + +**C2 late-backfill projector rolling (`PASS / FOUR-ROW REPAIR PENDING`, 2026-09-02).** The approved rolling packet completed one replica at a time: `projector_v2`, `projector_v2_2` and `projector_v2_3` now each run immutable image `sha256:b335d609db7b55000cba7cce455a525dbc8d7793faa83b94bc2cfbaa12f64430`, report `running=true`, `OOMKilled=false` and `RestartCount=0`. During the expected cooperative assignment transition, replicas two and three each logged a bounded `Kafka assignment changed before stable checkpoint` reconnect; they subsequently returned to serving without a process restart or fatal traceback. No other role was recreated and no V1, Kafka topology/offset, Redis, SQLite, runtime/TLS mount, Trading System, alpha or order-path state was changed. Next is a no-write provider-backed dry-run for the same named DOGE `1h` binding fenced at exactly four remaining rows; only that exact result permits the four-row normal data-plane repair. + +**C2 late-backfill four-row repair (`FAIL-CLOSED / REPLAY LINEAGE INSPECTION`, 2026-09-02).** The replacement immutable client dry-run returned exactly four provider-confirmed missing DOGE `1h` opens (`window_rows=10,000`, no production mutation). The explicit four-row apply was then allowed to use the normal raw Kafka path and a disposable non-root audit client. It did not converge after its fixed `180s` wait: the audit client exited `1`, never OOM-killed, with `remaining={"binance-usdm-dogeusdt-bar-1h":4}`. A follow-up read-only provider/cache probe independently returns the same four. This is not accepted as a successful repair and C2 remains blocked. The physical headroom rollout is live, so the remaining likely lineage is that already-canonical historical raw identities are deduplicated upstream and therefore are not re-emitted to projectors after the cache capacity correction. Next scope is read-only trace of the exact canonical/projector replay path and existing bounded replay tooling; no retry lottery, manual gap clearing, synthetic/revisioned data, V1 mutation, broad cache reset, Kafka topology mutation, Redis/SQLite deletion, Trading System, alpha or order-path mutation is allowed. + +**C2 bounded Rust dedup-generation recovery (`APPROVED / IN PROGRESS`, 2026-09-02).** Read-only code/runtime trace confirms the four rows are blocked by the active Rust cores' in-memory canonical event-ID deduplication: re-publishing the same provider-backed recovery rows cannot emit a new canonical Kafka record while the core generation remembers their stable BAR identities. This is intentionally distinct from durable idempotency: a fresh Rust generation deterministically canonicalizes the same captured raw rows, while the canonical SQLite/gateway layer accepts only records absent from its physical partition window and still suppresses all retained duplicates. The narrow recovery packet rolling-recreates only existing `rust_core`, `rust_core_2` and `rust_core_3`, one at a time, with their current immutable image `sha256:c63d54f05cd00407f7440e48d10d13daf3beea581cc624cd5c748e88aaf15ee0`, unchanged sealed runtime/TLS/state mounts and unchanged shared Kafka group. Each replica must return `running=true`, `OOMKilled=false`, `RestartCount=0` before the next moves. No source/config change, service/topology addition, offset reset, cache/Redis/SQLite deletion, V1, query/stream/projector image, Trading System, alpha or order-path mutation is authorized. Then one exact four-row provider-backed repair repeats through raw Kafka -> fresh Rust generation -> existing canonical/projector path. Exit requires coverage zero on a fresh read-only probe; otherwise it stops fail-closed. Rollback is only rolling the same three cores back to their identical current image/runtime, so no code or durable state rollback is needed. + +**C2 Rust recovery image preflight (`PASS / PIN REQUIRED`, 2026-09-02).** The sealed environment still names old Rust image `sha256:3056cf849d4d767f19431af92b944698b4dbef15c044942831619d296f8cd156`, while all three active cores are demonstrably the newer identical binary `qdl-v2-rust:2.0.0-0a122a8-l2hot@sha256:c63d54f05cd00407f7440e48d10d13daf3beea581cc624cd5c748e88aaf15ee0`. A naive Compose recreate would silently downgrade them. The recovery command must therefore override only `QDL_STABLE_RUST_IMAGE` to the active `c63d...15ee0` immutable digest, retaining every other sealed environment value and mount; this is a runtime provenance pin, not a bundle/config mutation. + +**C2 four-row replay root-cause correction (`PASS / STREAM-WRITER ROLLING REQUIRED`, 2026-09-02).** The bounded Rust generation roll completed safely but the post-replay audit still failed closed. Exact SQLite inspection then isolated the actual remaining defect: the DOGE canonical partition remains physically `10,000` rows (`logical_offset 1019..11018`) even though projectors run the `10,064`-headroom image. The canonical SQLite writer is `stream_v2_active`/`stream_v2_passive`, not the projector; both streams still run the prior Python artifact and therefore retain the old trim bound. The earlier Rust rolling was safe (`RUST_PRIMARY`, same pinned `c63...15ee0`, no OOM/state mutation) but did not address the writer ownership boundary. Correct runtime scope is now a rolling recreate of exactly `stream_v2_passive` then `stream_v2_active` onto immutable Python image `b335d609db7b55000cba7cce455a525dbc8d7793faa83b94bc2cfbaa12f64430`, retaining runtime/TLS/state/Redis/Kafka and every other role. Passive must be healthy before active moves; active lease/failover must remain available. No query, V1, core, ingestor, bar-edge, projector, offsets, cache reset/deletion, Trading System, alpha or order mutation is authorized. Once both writers expose the corrected shared spool capacity, the exact four-row provider repair is repeated once and must make the same read-only coverage probe report zero missing. + +**C2 bounded DOGE repair scope refresh (`PASS / FIVE-ROW APPLY AUTHORIZED`, 2026-09-02).** The stream-writer rollout completed passive first then active: both now run `b335d609db7b55000cba7cce455a525dbc8d7793faa83b94bc2cfbaa12f64430`, remain `healthy`, `running=true`, `OOMKilled=false` and `RestartCount=0`. The exact no-write provider/cache dry-run correctly stopped rather than widening its prior four-row fence: because the rolling ten-thousand-row history window advanced by one closed hourly BAR during remediation, it now observes five authentic missing DOGE opens, precisely `2026-09-02T09:00:00Z`, `10:00:00Z`, `11:00:00Z`, `12:00:00Z`, and `21:00:00Z`. This is the same one binding and provider-backed recovery window, not a symbol/venue expansion. The next bounded action is one apply with `expected_missing=5`, a fixed `180s` convergence wait and a read-only zero-coverage verification. It may emit only those normal final-BAR recovery writes through the existing raw -> Rust -> canonical -> projector path; it does not change offsets, delete/flush cache state, touch V1, Trading System, alpha or order state. Any count change again stops fail-closed. + +**C2 late-backfill coverage-reader correction (`IN PROGRESS / SOURCE-ONLY`, 2026-09-02).** The five-row provider-backed apply reached the live canonical SQLite writer: the DOGE `1h` partition grew to `10,005` physical rows and contains five real `FINAL` BAR envelopes at the exact approved openings. The client still timed out with four remaining because `StableBinanceBarEdge._durable_final_bar_opens()` reads only `LIMIT 10,000`; its own new physical late-backfill headroom therefore makes the first four retained rows invisible to the coverage reader. This is a narrow shared contract defect between physical spool retention and the reader, not a provider gap, Rust dedup fault, projector failure, query/stream consumer fault, or evidence of lost repair data. The source correction must centralize public (`10,000`) and physical (`10,064`) BAR window constants, retain the public API ceiling unchanged, and make durable coverage inspect the physical window. Required tests prove no caller can request more than `10,000` public BAR rows, while coverage sees a bounded late-backfill tail and is not confused by non-BAR or mismatched identity. Then build one immutable Python image, rolling-recreate only `binance_bar_edge` with existing mounts, use that image for a read-only zero-coverage verification, and only then resume C2. No cache reset, re-publish, V1, Kafka topology/offset mutation, Redis/SQLite deletion, Trading System, alpha or order-path change is allowed; the five raw/canonical writes already committed are preserved as real evidence. + +**C2 late-backfill coverage-reader source gate (`PASS / IMMUTABLE EDGE IMAGE REQUIRED`, 2026-09-02).** `qdl.runtime.stable_capacity` is now the single source for the public `10,000` BAR contract and the bounded `10,064` physical spool window. `stable_spool_capacity()` and `StableBinanceBarEdge` consume the same shared constants; durable final-BAR coverage binds the physical window as a SQLite `LIMIT` parameter, while public request validation remains capped at `10,000`. The new regression proves the reader requests the physical tail and still recognizes only an identity-matched final BAR; existing provider/gap/mismatch regressions remain in the same matrix. The isolated, network-disabled, read-only UID-`10001` matrix passed `55/55` in `11.763s`: `test_phase533_query_readiness`, `test_phaseb_bar_history_bootstrap`, `test_c419_fast_final_bar_delivery`, and `test_phase115c_bar_edge_schedule`. No runtime role or data state changed during this source gate. Next: commit this source slice, build one immutable Python image, rolling-recreate only `binance_bar_edge`, read-only verify zero missing for the already-persisted five provider BARs, then resume C2 without re-publishing them. + +**C2 coverage-reader edge rollout (`APPROVED / IN PROGRESS`, 2026-09-02).** Source commit `9e3a99f` was built as immutable `qdl-v2-python:2.0.2-9e3a99f@sha256:2ab6a946e5ca59e234063fb661a7a7738795dab124e371b5579db31febfd42e0`; the no-source-mount, network-disabled, read-only UID-`10001` image matrix passed `55/55` in `12.326s`. The runtime packet rolling-recreates exactly `binance_bar_edge` with this image, retaining its existing runtime/TLS/state/Redis/Kafka mounts, service name, network and authority record. The exact rollback coordinate is the current `qdl-v2-python:2.0.1-43faf3d@sha256:6090b3a6c1c6bc431a329ab85cad7fe61750a33dbed3a5bb2d264c532f211545` with the same mounts. No stream/query/projector/core/ingestor/V1 role, Kafka topology or offsets, Redis/SQLite reset/deletion, Trading System, alpha or order path is in scope. After the role returns running without OOM/restart, a disposable read-only coverage probe must report zero missing for the five already-canonical provider BARs; no re-publish is permitted. + +**C2 edge rollout manifest correction (`PASS / RECOVERY ROLLING REQUIRED`, 2026-09-02).** The first edge recreate failed before its event loop because the command omitted the pre-existing runtime overlay `/home/bobby/.local/state/qdl-v2/phase543-reader-binding-20260902/bar-edge-history-convergence-20260902T1815Z/bar-edge-history.override.yml`. Without it, Compose fell back to legacy `stable-crypto-bar-edge.json` (`state.v1`, four obsolete bindings), which the current V4-only runtime correctly rejects. The attempted same-image rollback repeated that command omission and also stopped before any provider request, Kafka publish, checkpoint persist or data-plane mutation. Read-only compatibility probes prove the overlay's exact V4 checkpoint `/var/lib/qdl-stable/runtime/phase54-alpha-demand-5edbc8c.history10000-20260902.json` matches current catalog/acquisition `8/16`, warmup `10,000`, canonical cache identity and 140 BAR bindings; it restores with 130 already checkpointed and preserves the existing bounded bootstrap behavior for the ten newly demanded OKX SOL BAR bindings. Correct recovery is a one-role recreate using the existing history overlay plus a disposable image-only overlay for `2ab6...42e0`. The image-only file is removed immediately after the service starts. This restores the original runtime configuration rather than copying/deleting a checkpoint or creating a new state namespace. + +**C2 edge recovery checkpoint selection (`PASS / ONE-ROLE RECOVERY`, 2026-09-02).** The history-`10000` checkpoint attempted the ten uncheckpointed OKX SOL bindings; its truthful provider history returned an authentic time gap and the edge retried fail-closed. The role remained running, but cannot progress while that incomplete bootstrap blocks the shared loop. A separate existing V4 checkpoint `phase54-alpha-demand-5edbc8c.json` is a strict match for the same current catalog/acquisition/cache identity and all 140 binding IDs; a disposable no-loop probe with its matching `QDL_STABLE_BAR_WARMUP_ROWS=1000` restored all 140 watermarks and issued a new generation successfully. `1000` is the edge's bootstrap retention parameter only; it does not reduce the public `10,000` request ceiling or the explicit provider `FRESH_SNAPSHOT` coverage contract. The recovery overlay therefore selects this complete real checkpoint and sets only the matching bootstrap value; current `10,000` bounded catch-up remains unchanged. This avoids fabricated provider history and prevents one known native long-history gap from blocking already-certified Binance/OKX BAR lanes. + +**C2 durable-coverage reader runtime exit (`PASS / ONE REAL C2 RECEIPT PENDING`, 2026-09-02).** The recovered `binance_bar_edge` now runs immutable `qdl-v2-python:2.0.2-9e3a99f@sha256:2ab6a946e5ca59e234063fb661a7a7738795dab124e371b5579db31febfd42e0` with the matching existing V4 checkpoint `phase54-alpha-demand-5edbc8c.json`, `warmup_rows=1000`, the unchanged catalog/acquisition/cache identity and all `140` BAR bindings checkpointed. It is `running`, has no OOM kill/restart, restores the same real provider-derived watermark generation and continues normal final-BAR ACKs. A disposable no-write, authenticated provider/cache probe on the formerly blocked `binance-usdm-dogeusdt-bar-1h` reads the physical `10,064` spool window and returns `missing_rows=0`; it publishes nothing and confirms the five already-committed provider BARs are visible to the corrected coverage reader. The next and only acceptance action is one fresh C2 four-identity, 299-product, true-300-second no-order receipt using the existing V2 query/stream endpoints and V1 local fallback policy. It may read V2/V1 only; it has zero provider, Gateway, Risk, order or state-mutation authority. Rollback is immediate by stopping/removing only the disposable C2 client; V1, Kafka topology/offsets, Redis, SQLite, all runtime roles, Trading System, alpha and order paths remain unchanged. + +**C2 retry 20260902T223058Z (`FAIL-CLOSED / READ-ONLY FRESHNESS DIAGNOSIS`, 2026-09-02).** The immutable `2ab6...42e0` C2 client passed its `116/116` no-source-mount, network-disabled preflight matrix. Its first two launcher starts stopped before an endpoint request because copied wrapper permissions omitted, respectively, world read/execute for the container mount and the documented bootstrap `--user 0:0`; both were corrected only on the disposable evidence scripts. The actual client then dropped to UID `10001`, `NoNewPrivs=1`, all effective/permitted/inheritable/ambient capabilities zero, and made no provider, Gateway, Risk, order or state mutation. It correctly stopped at `trading-system.paper.stable / OKX.SWAP.PERPETUAL.ETH-USDT / BAR 1m` with typed `required data exceeds its freshness policy`; it did not reach the 300-second observation. This is a real V2 quality decision, not a DOGE coverage regression or a basis for widening SLA. Next scope is read-only typed status, cache/watermark and bar-edge lineage for the five OKX liquid symbols on both query replicas; only a proven shared projection/provider defect may be repaired, then one new C2 receipt is permitted. + +**C2 OKX final-BAR projection attachment (`APPROVED / CONFIG-ONLY RUNTIME REPAIR`, 2026-09-02).** Read-only deployed-config inspection proves the C2 failure is a missing overlay, not a provider, quality-policy or Rust-core defect. The sealed V4 projection `/runtime/phase54-alpha-demand-5edbc8c/{catalog,acquisition}.yaml` contains all `140` binding identities and deliberately projects every enabled OKX Swap final BAR to the bounded Python provider-finality edge (`PYTHON_REST`), while the image-default acquisition plan declares the same routes `RUST_NATIVE`. The active edge retained only its state/warmup override and therefore read the image-default plan; it ACKed Binance only, leaving `OKX ETH 1m` stale. The corrected packet recreates only `binance_bar_edge` with immutable `2ab6...42e0` and the existing sealed `catalog.yaml`, `acquisition.yaml`, complete V4 checkpoint `phase54-alpha-demand-5edbc8c.json`, `warmup_rows=1000` and unchanged `max_catchup_rows=10000`. It permits normal authentic final-BAR writes through the existing raw -> Rust canonical -> projector/cache plane. It excludes V1, Kafka topology/offsets, Redis/SQLite reset/deletion, Rust/ingestors/projectors/query/stream, Trading System, alpha and order path. Rollback is one recreate of only the same edge with its prior image/config; no durable state is deleted. Exit requires live real-provider ACKs for both Binance and OKX final BAR lanes, no OOM/restart, and a fresh C2 receipt. + +**C2 sealed-BAR config hardening (`APPROVED / SOURCE + ONE-ROLE CONFIG ONLY`, 2026-09-02).** The source Compose contract will expose `QDL_STABLE_SOURCE_BINDINGS` and `QDL_STABLE_ACQUISITION_BINDINGS` as explicit optional runtime overrides with the current image paths as safe defaults. This does not change any default source deployment; it prevents a future bounded edge recreate from silently discarding an already-sealed projection. The one-role runtime override supplies the existing V4 catalog/acquisition paths plus state/warmup/image provenance. Gates are Compose render with the sealed env, stable catalog/acquisition validation and the focused final-BAR/C2 regression matrix before the sole edge recreate. Rollback is removal of the overlay/variables and the prior one-role image/config only. No bundle secret is printed or committed. + +**C2 source/runtime projector-bound reconciliation (`APPROVED / SOURCE-ONLY`, 2026-09-02).** The focused Compose contract test correctly caught an unrelated source drift before rollout: all three live projectors have `QDL_STABLE_MAX_PENDING_RECORDS=2048` under their approved `512 MiB` bound, while the current source Compose file had regressed to `1024` in all three service stanzas. The runtime is healthy; no projector is being recreated. The source must restore `2048` so the next ordinary deployment cannot silently halve the tested bounded backlog. This is a three-line declarative reconciliation, covered by the existing Compose isolation/capacity test; it does not widen memory, queue bytes, topology, V1, data plane or consumer authority. + +**C2 source/runtime projector-byte reconciliation (`APPROVED / SOURCE-ONLY`, 2026-09-02).** The same contract gate then identified the paired source drift: live projectors retain the approved `QDL_STABLE_MAX_PENDING_BYTES=33554432`, while all three source stanzas declared `16777216`. Restore the existing `32 MiB` tested limit in source together with the `2048` record bound. No runtime role changes, no memory limit increase, and no data-plane/consumer mutation is permitted. + +**C2 sealed-BAR mount correction (`FAIL-CLOSED / ONE-ROLE RECOVERY`, 2026-09-02).** The first sealed-projection recreate stopped before its event loop, provider request, Kafka publish or checkpoint write because the generic Compose environment mounted `/runtime` from `phase103...`, whereas the existing sealed BAR projection is under `session-liveness-43cdbe3.../runtime`. The container is cleanly `exited`, `restart=0`, `OOMKilled=false`; no data-plane mutation occurred. Recovery is still one role only: retain the same `2ab6...42e0` image and sealed override, but set only the Compose host `QDL_STABLE_RUNTIME_DIR` for this edge recreate to the existing session-liveness runtime directory. This makes the already-declared `/runtime/phase54-alpha-demand-5edbc8c/{catalog,acquisition}.yaml` paths resolvable. All other service mounts remain untouched because `--no-deps` targets only the edge. Rollback remains the prior one-role image/config. + +**C2 sealed-BAR mount recovery exit (`PASS / C2 RETRY READY`, 2026-09-02).** The one-role recovery is now `running` on immutable `2ab6...42e0`, `restart=0`, `OOMKilled=false`, restores the matched 140-binding V4 checkpoint and produces real provider final-BAR ACKs for both venue lanes. The first catch-up ACK contains the enabled OKX Swap BAR identities across BTC/ETH/SOL/DOGE/BNB; subsequent real `1m` ACKs include `okx-swap-eth-usdt-swap-bar-1m` and the other four liquid OKX swaps alongside their Binance USD-M peers. These are normal authentic provider -> raw Kafka -> Rust canonical -> projector/cache writes, not synthetic repair data. The sealed source/acquisition path is therefore the required stable edge contract. A fresh four-identity C2 no-order client will use a new evidence namespace; its inherited V1 fallback policy and all no-order exclusions remain unchanged. + +**C2 final no-order receipt (`IN PROGRESS`, 2026-09-02).** A prior C2 attempt +stopped fail-closed before its 300-second observation at an OKX BAR freshness +read; the root cause was the missing sealed acquisition projection now repaired +by the one-role edge recovery above. Two disposable diagnostics were used only +to validate the launcher topology and were not acceptance: the one-network +probe correctly could not resolve the executor-network stream aliases, while +the two-network probe demonstrated the full scope needs the declared 300-second +window rather than a shortened diagnostic deadline. Both made no provider, +Gateway, Risk, order or persistent-state mutation and are excluded from +evidence. The one valid next operation is exactly one new four-identity, +299-product, 300-second C2 client attached to the existing stable-internal and +executor networks. It is read-only, non-root after bootstrap, has no order or +provider authority, uses immutable image `2ab6...42e0`, and self-removes after +writing only bounded receipt evidence. V1 remains the governed rollback path; +all runtime services, Kafka/Redis/SQLite state, Trading System and alpha remain +unchanged during this receipt. + +**C2 final launcher correction (`FAIL-CLOSED / RETRY READY`, 2026-09-02).** +The first final-client creation inherited the immutable image's normal Uvicorn +entrypoint rather than explicitly running `/bin/sh /bootstrap-c2.sh`. Its +read-only root filesystem made that mistake fail before bootstrap, identity +copy, endpoint, provider, stream, Gateway, Risk, order or state activity; the +exited disposable container was removed. The retry changes only the test +container entrypoint/command to the already-proven bootstrap pair. It keeps the +same two existing networks, image, read-only inputs and bounded evidence path. + +**C2 DOGE 1h physical-tail reader diagnosis (`FAIL-CLOSED / SOURCE REPAIR`, +2026-09-02).** The corrected final client reached V2 with its governed alpha +identity and stopped at `BINANCE.USDM.PERPETUAL.DOGE-USDT / BAR 1h` because the +warmup returned `OPEN_SEQUENCE_GAP`. A read-only typed-status probe proves the +latest tail itself is `LIVE`, complete, gap-free and execution-eligible; the +warmup failure is therefore not a provider/session/SLA issue. A no-network, +read-only audit of the canonical SQLite partition found `10,006` unique, +market-time-contiguous final BAR opens and no duplicate payloads. The defect is +the shared reader's contradictory retention view: the spool correctly retains +the bounded physical `10,064` rows for authentic late backfills, but +`StableSpoolQueryBackend` still scans only `10,000` rows by logical append +offset. Six valid late-backfill rows fall just outside that scan even though +their market times lie in the public warmup tail; the reader then correctly +fails closed on the artificial gap. + +The in-scope repair is one shared source change: every internal BAR scan used +to construct history, latest quality, stored events and gap status must read +`STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW`, then sort/select the caller's market +time window and retain the unchanged public maximum of `10,000` rows. It must +not widen any public API/SDK quota, clear a gap manually, alter providers, +Kafka, Redis, SQLite, V1, Trading System, alpha or order behavior. Regression +must cover a physical tail containing late valid BARs whose logical offsets are +old enough to fall outside a `10,000` append-tail scan, plus normal gaps and +public-limit invariants. After source/image gates, only existing V2 query and +stream readers may be rolled to consume the fix; a new full C2 receipt is then +the sole acceptance action. + +**C2 physical-tail reader source gate (`PASS / IMMUTABLE READER HANDOFF +READY`, 2026-09-02).** The repair centralizes the distinction between the +public BAR ceiling and the small retained physical headroom. `SQLiteDurableSpool` +now permits an internal tail read up to the explicit configured partition +capacity only; default/public callers remain capped at `10,000`. The shared +stable reader uses the physical `10,064` window only for BAR latest/history/ +quality/gap construction, then sorts by market time and returns the caller's +unchanged bounded public result. Book and non-BAR paths retain their existing +public bounds. A regression materializes `10,064` authentic-shaped rows where +64 recent market-time repairs have old append offsets: a 700-BAR public warmup +is ordered, full and gap-free, while a real missing bar remains fail-closed. + +The isolated source gate used the existing immutable +`qdl-v2-python@sha256:2ab6a946e5ca59e234063fb661a7a7738795dab124e371b5579db31febfd42e0` +with the candidate source mounted read-only, UID/GID `10001`, a read-only root, +ephemeral `/tmp` and `--network none`. It passed **117/117** with one explicit +skip in `33.853s`: stable reader/edge, durable transport, bounded bar-edge +projection, fast final-bar delivery, C2 consumer acceptance and stable +deployment. No runtime role, provider, Kafka, Redis, SQLite, V1, Trading +System, alpha, Gateway/Risk or order path changed. `git diff --check` passed. + +**Next bounded operation.** Commit this source/documentation slice, build one +canonical immutable Python reader image from that commit, rerun the same +no-source-mount matrix, then rolling-recreate only `stream_v2_passive`, +`stream_v2_active`, `query_v2_1` and `query_v2_2` with their exact current +images retained as rollback coordinates. After healthy/restart-free checks, +run exactly one fresh four-identity C2 no-order receipt. V1 and every other +role remain unchanged. + +**C2 physical-tail reader handoff packet (`OWNER-APPROVED / PRE-ROLL`, +2026-09-02).** Commit `f9c27e7` is sealed as +`qdl-v2-python:2.0.3-f9c27e7@sha256:c13c2a9d7d39f2fcac29c65bb57b16e0dfc9132563a1518c945db20006455386`. +Its no-source-mount, read-only, UID/GID-`10001`, `--network none` matrix passed +**117/117** with one explicit skip in `32.480s`. The bounded runtime handoff +may recreate exactly four existing cache readers in dependency-safe order: +`stream_v2_passive`, `stream_v2_active`, `query_v2_1`, then `query_v2_2`. +It must reuse the complete active Compose configuration set and env file, +preserving each role's TLS/runtime/state mounts, network attachments, identity, +ports, resource limits, volumes and service name; an image-only override is +the sole rendered configuration change. + +The exact rollback images are +`qdl-v2-python@sha256:b335d609db7b55000cba7cce455a525dbc8d7793faa83b94bc2cfbaa12f64430` +for both stream roles and +`qdl-v2-python:2.0.1-d9dea34@sha256:dcb15154ccff53490d152152af99a8aa808c9901c5d210135d791d74d0398bd7` +for both query roles. Before each recreate, render/inspect must demonstrate +that only `image` differs. After each, require `healthy`, zero new restarts, +`OOMKilled=false` and no bounded error signature. V1, Kafka topology/offsets, +Redis, SQLite, bar-edge, Rust, ingestors, projectors, Trading System, alpha, +Gateway/Risk and order paths are excluded. On any failure, recreate only the +failed role with its exact retained rollback image and stop this packet. + +**Acceptance boundary.** A healthy four-role roll does not count as C2. It +only unlocks one fresh, disposable four-identity, 299-product, 300-second +no-order receipt with governed V2 reads and declared V1 fallback drill. The +client has no provider, Gateway, Risk, order or durable-state mutation +authority and is removed after bounded evidence is captured. + +**C2 reader handoff render gate (`PASS / ROLL AUTHORIZED`, 2026-09-02).** The +complete active Compose file chain and current `stable.env` were rendered both +with and without the new final image override. For +`stream_v2_passive`, `stream_v2_active`, `query_v2_1` and `query_v2_2`, a +canonical hash of the rendered service after removing `image` is identical; +the only configuration diff is the new immutable digest. The preflight records +four non-image hashes in the bounded handoff evidence. Runtime inspection also +found an old Compose history artifact: the active stream containers still use +`b335...4430` while the rendered historical base says `d9dea...8bd7`; their +mounts, networks, identities and runtime are unchanged. The rollback overlay +therefore pins the *actual* active stream digest `b335...4430` and actual query +digest `d9dea...8bd7`, rather than relying on stale Compose history. This is +provenance hardening only; no role, data plane or topology changed during +rendering. + +**Fresh C2 namespace (`OWNER-APPROVED / EXECUTING`, 2026-09-02).** The +four-role handoff completed in declared order on `c13c...5386`; every role is +running, healthy, `restart=0`, `OOMKilled=false`, and a bounded five-minute +error scan is empty. One new disposable C2 client now uses this reader image, +the pre-existing Phase-10.3 runtime authority mount, the existing V2 stable +internal and executor networks, and only four governed identities. It runs the +unchanged 299-product, 300-second no-order receipt script with signed cursor +replay/reconnect and declared V1 fallback drill. It cannot reach a venue +provider directly and has no Gateway, Risk, order or persistent-state mutation +credential. Any nonzero exit is fail-closed; the client is removed and the four +reader roles stay on `c13c...5386` unless their own health changes. + +**C2 launcher capability correction (`FAIL-CLOSED / RETRY READY`, 2026-09-02).** +The first new client exited before it could create `/tmp/inputs`: the outer +Docker launcher dropped all capabilities before `/bootstrap-c2.sh`, so its +intentional `setpriv --reuid=10001` failed with `setresuid: Operation not +permitted`. Query replica logs confirm zero endpoint request, and no receipt, +provider, Gateway, Risk, order or durable-state action exists. The previous +approved C2 topology correctly starts bootstrap as root with only +`no-new-privileges:true`; bootstrap then drops UID/groups/capabilities before +copying identities and before running the Python client. The fresh retry will +match that proven topology exactly, retain no additional capability after the +drop, use a new evidence namespace and remove its container on exit. This is a +launcher repair only; no V2 role is recreated and no policy/SLA is relaxed. + +**C2 historical replay watermark diagnosis (`FAIL-CLOSED / SOURCE-ONLY +ACCEPTANCE REPAIR`, 2026-09-02).** A disposable governed DOGE `1h` trace proved +the reader/runtime is not gapped: strict V2 warmup returned watermark `11024`; +the deliberate historical seed returned `10004`, then signed stream replay +advanced contiguously through `10005..10037` before the bounded diagnostic +stopped. The C2 harness incorrectly assumes all authentic late/backfill events +after a historical seed fit its arbitrary 16-event observation cap and demands +that this test-only cursor reach the *current* watermark. That is not the +production SDK path: real consumers take a strict current warmup cursor first, +whereas the C2 historical seed exists solely to prove cursor/reconnect without +waiting for a future 1h close. + +The repair is narrow and preserves stricter domain checks: for a historical +alpha BAR, require one monotonic signed replay event across query/stream +replicas, then require a new strict current V2 snapshot using the original +governed requirement. It must not drain arbitrary historic offsets, relax +freshness/gap/finality, use a provider/V1 substitute, or change reader/runtime +services. Regression will cover the real shape (`seed=10004`, first=`10005`, +strict=`11024`) and prove it passes only after strict current readback; failure +of that readback remains fail-closed. After source/image tests, build a +disposable acceptance-client image only and run one fresh C2 receipt. The +already-rolled reader roles remain on `c13c...5386`. + +**C2 historical replay contract source gate (`PASS / DISPOSABLE CLIENT IMAGE +REQUIRED`, 2026-09-02).** The harness now treats the historical BAR seed as +what it is: a bounded signed-cursor/reconnect proof, not an unbounded backlog +consumer. It requires exactly one monotonic replay frame across replicas and +then a strict current V2 snapshot using the original governed requirement. +The regression reproduces the real `10004 -> 10005 -> 10006` seed/replay shape +against strict watermark `11024`, proves later historical offsets are not +drained, and fails if the current snapshot cannot be read. The isolated, +read-only, non-root, no-network C2 matrix passed **68/68 in 15.362s** across +receipt replay, identity, fallback and release-certification cases. No runtime +reader role, provider, Kafka, Redis, SQLite, V1, Trading System, alpha, +Gateway/Risk or order path changed. Next: commit this source slice, build one +immutable disposable acceptance-client image, run its no-source-mount matrix, +then run exactly one fresh 299-product/four-identity C2 receipt. The current +reader runtime remains `c13c...5386` throughout. + +**C2 stale-first-frame diagnosis (`FAIL-CLOSED / SOURCE-ONLY ACCEPTANCE +REPAIR`, 2026-09-03).** The replay-revalidation C2 progressed beyond DOGE BAR +and then stopped at Trading System's `OKX BTC QUOTE` when its first stream +frame was stale under the execution freshness policy. A governed read-only +trace immediately afterward proves the actual route is healthy: V2 snapshot +freshness `227ms`, typed status `235ms`, first stream frame `134ms`, all +`LIVE`, complete, gap-free and execution-eligible at `3136504 -> 3136505`. +This identifies an admissible replay race, not an OKX outage: a delayed frame +can arrive after a fresh snapshot cursor and must never be treated as an +execution price. + +The C2 harness must therefore handle only a first-frame `DATA_STALE` as +replay-only state, then immediately require/validate a new strict current V2 +snapshot before accepting cursor continuity. `OPEN_SEQUENCE_GAP`, identity, +session, finality, source-authority and a failed strict read remain fail-closed; +no SLA or provider data is altered. Regression must cover stale-first/fresh +readback and stale-first/stale-readback rejection. This changes acceptance +semantics only, not SDK runtime delivery or any V2 reader service. After the +source/image matrix, one fresh C2 receipt is again permitted. + +**C2 replay-revalidation image gate (`PASS / FINAL RECEIPT READY`, +2026-09-02).** Commit `7cd0265` is sealed as +`qdl-v2-python:2.0.4-7cd0265@sha256:8099b83488fb583e9d06c8021e2195c0724a12107ea921c2caf9d269741dd0cd`. +The image-contained, no-source-mount matrix ran read-only as UID/GID `10001` +with `--network none` and passed **178/178** with one explicit skip in +`45.342s`. It combines the physical-tail reader/transport/bar-edge matrix with +the C2 replay, identity, fallback and release-certification suite. The next +operation is exactly one fresh C2 four-identity/299-product/300-second receipt +using this image only as a disposable client; it does not recreate the already +healthy reader roles on `c13c...5386`. + +**C2 stale-first-frame contract source gate (`PASS / IMMUTABLE CLIENT REBUILD +REQUIRED`, 2026-09-03).** The acceptance harness now distinguishes a delayed +first durable stream frame from a current execution price. Only a first-frame +`DATA_STALE` outside historical BAR replay may be projected as `state_replay`; +it is acknowledged solely for signed-cursor continuity and is immediately +followed by a strict current V2 snapshot under the original requirement. The +current read must pass normal freshness, session, gap, identity, finality and +source-authority validation before resume proceeds. `OPEN_SEQUENCE_GAP`, every +non-stale error, and a stale current read still fail closed. New regressions +prove both the fresh-readback success and stale-readback rejection. The +isolated source-mounted Docker matrix ran `64/64` in `5.985s` as UID/GID +`10001`, read-only, capability-dropped and network-disabled. No runtime role, +provider, Kafka, Redis, SQLite, V1, Trading System, alpha, Gateway/Risk or +order path changed. Next is one immutable disposable client build, a +no-source-mount regression, then one fresh 299-product/four-identity/300-second +C2 receipt; readers remain on `c13c...5386`. + +**C2 stale-first-frame immutable-client gate (`PASS / ONE REAL RECEIPT +AUTHORIZED`, 2026-09-03).** Commit `a497ab0` is sealed as +`qdl-v2-python:2.0.5-a497ab0@sha256:08f43a67393cea317fff3765cc8b369d49e1378cb25f0644c2243f35f3ed5c0c`. +Its OCI revision/version labels exactly match `a497ab0` / `2.0.5-a497ab0`, and +the image runs as `qdl:qdl`. The image-contained, no-source-mount regression +ran with a read-only root, UID/GID `10001`, all capabilities dropped, +`no-new-privileges`, an ephemeral `/tmp` and `--network none`; it passed +**193/193** with one explicit skip in **41.359s**. Coverage includes the +physical-tail durable reader/transport/bar-edge paths, C2 receipt and cursor +reconnect, governed identity/fallback policy and release-observation contracts. +No V2 reader role is recreated for this client-only change. The sole runtime +operation now permitted is one fresh, disposable, four-identity, 299-product, +true-300-second C2 no-order receipt. It may read the existing V2 query/stream +and governed V1 fallback only, has no provider, Gateway, Risk, order, signal, +sizing, Redis/Kafka/SQLite or Docker authority, and is removed after terminal +result. The rolled readers stay on `c13c...5386`; V1 is unchanged. + +**C2 consumer-quota diagnosis (`FAIL-CLOSED / SOURCE-ONLY ACCEPTANCE +SCHEDULING REPAIR`, 2026-09-03).** The single authorized `a497ab0` receipt +started with the declared four identities and 299 products, read only the +existing V2 query/stream plane and local governed V1 fallback, then stopped at +`alpha.binance.paper.stable / BINANCE.USDM.PERPETUAL.DOGE-USDT / BAR 4h`. +The historical signed cursor correctly expired into the SDK's required fresh +snapshot path; that strict V2 request received `RATE_LIMITED: consumer request +quota is exhausted`. The disposable client exited `1`, was not OOM-killed, +recorded no receipt and made no provider, Gateway, Risk, order, signal, sizing +or durable-state mutation. Its launcher dropped to UID/GID `10001` with +`NoNewPrivs=1` and every effective/permitted/inheritable/ambient capability +zero before the read loop. + +This is a C2 harness scheduling defect, not a source-data, provider, +freshness, reader, cursor, session or fallback defect. The harness starts all +durable products concurrently while each alpha identity is correctly bounded +to `180` REST requests/minute. A normal product proof performs multiple +query/warmup/read-back calls, so the full 125-route alpha scope can exceed its +own declared minute budget before C2 reaches its real 300-second observation. +The repair remains source-only and preserves the whole 299-product proof: + +1. derive one shared request budget from each sealed consumer manifest and + pace every C2 REST request across both query replicas below that consumer's + real limit, after a fresh wall-clock quota window; +2. keep opening proof for every durable route: primary/secondary typed reads, + signed cursor/reconnect, strict current read-back and the governed fallback + drill; +3. replace only the closing duplicate stream sweep with a full, per-consumer + V2 `warmup:batch` revalidation on both replicas, preserving strict identity, + finality, quality, cardinality and replica checks for every declared route; +4. record opening, observed-300-second and closing durations separately. The + acceptance is bounded by an explicit opening deadline; the 300-second + observation begins only after the full opening proof, rather than treating + an incomplete opening timeout as evidence. + +The repair must fail closed for any real `RATE_LIMITED`, identity, quality, +gap, finality, cursor, reconnect or cardinality failure. It must not raise +runtime quotas, relax freshness/SLA, reduce the 299-product scope, generate +data, alter any deployed role, or use V1 as a V2 substitute. Regression gates +cover per-identity quota pacing, clean-window alignment, opening cardinality, +full batch closing cardinality/replica validation, and the existing +stale-first/cursor/fallback failures. After a commit and immutable disposable +client image matrix, exactly one new C2 receipt is allowed; the failed client +is then removed as scoped test cleanup while its compact failure digest remains +traceable in evidence. + +**C2 quota-aware opening and batch-closing source gate (`PASS / DISPOSABLE +CLIENT IMAGE REQUIRED`, 2026-09-03).** The acceptance harness now loads the +real per-identity request-per-minute quota from the sealed release manifest and +wraps only its disposable REST query transport in a shared identity pacer. +Both query replicas, warmups, status retries and reference batches therefore +share the same bounded allowance; V1 fallback remains the separately governed +local read drill. C2 first aligns to a fresh Redis minute window, uses at most +75% of the declared quota, and records the exact request count and pacing wait +per identity. Production quota values, SDK construction, query/stream service +runtime and all public contracts are unchanged. + +The opening continues to prove every non-on-demand product individually: +primary/secondary typed read, signed cursor/reconnect, strict current read-back +and declared V1 fallback-return. After that opening, the actual 300-second +observation begins. The closing boundary now revalidates the complete scope +through typed V2 `warmup:batch` calls against both replicas, bounded by each +manifest's `max_batch_items`; it validates every item identity, source, +quality, gap/finality, history/latest view, cardinality and replica alignment. +It does not open a second copy of every stream. Opening, observation and +closing each have explicit bounded clocks (`900s`, `300s`, `120s` defaults) +and evidence fields; any quota, partial batch, stale/gap, cursor, reconnect or +identity error remains terminal. + +The reusable Phase-10.3 receipt primitive accepts an optional acceptance-only +client factory, preserving its default SDK client exactly. The C2 harness uses +that seam solely to pace itself; normal consumer and service code cannot enter +this path. Focused no-network/read-only UID-`10001` regression passed **64/64 +in 10.787s**, including deterministic minute alignment/pacing, both-replica +batch closing for every test product and missing-cardinality rejection. The +expanded reader/transport/bar-edge/C2 matrix contains **196** cases and exited +successfully under the same offline, read-only container controls. `git diff +--check` passed. No runtime role, provider, Kafka, Redis, SQLite, V1, Trading +System, alpha, Gateway/Risk or order path changed. Next: commit this coherent +source slice, build one immutable disposable acceptance-client image, repeat +the no-source-mount matrix, then run exactly one quota-aware 299-product C2 +receipt. The failed `a497ab0` client container is test-only and will be removed +after its failure digest is retained in bounded evidence. + +**C2 quota-aware receipt (`FAIL-CLOSED / QUIET FINAL-BAR ACCEPTANCE REPAIR`, +2026-09-03).** Immutable client +`qdl-v2-python:2.0.6-8620303@sha256:a7a7cf86422955158c1e17002fece142814ce5fb6ac1c3a15b1359cf7d8778e2` +was built from source commit `8620303`, with matching OCI revision/version +labels. Its focused no-source-mount C2 image suite passed `15/15`; the client +was then launched once as the declared read-only, non-root-after-bootstrap +four-identity receipt. It ran for roughly four minutes, did not OOM, emitted no +receipt, made no provider/Gateway/Risk/order/signal/sizing/durable-state +mutation, and stopped at `alpha.binance.paper.stable / DOGE-USDT / BAR 4h`. +The bounded stderr digest is +`9167bb583160b245a02417bbfb5f36a4e29f960d226938a651c0a309ea112c90`. + +The failure is no longer `RATE_LIMITED`: the manifest pacing held. It is a +second C2-only semantic mismatch. A slow historical alpha BAR can have a valid +signed warmup cursor and receive only `REPLAYING`/`LIVE` controls during a +300-second window because no new closed `4h` bar exists. The existing harness +unconditionally calls `_next_data()` and rejects controls without a market +event, even though it can already prove a strict current, final, authoritative +BAR through V2. This must not be solved by generating a bar, changing a venue +SLA, weakening finality, or claiming a replay event that did not arrive. + +The narrow source repair is to admit a *quiet historical alpha BAR handoff* +only after both sessions prove the signed cursor controls and each immediately +passes a strict current V2 warmup/snapshot under the original requirement. Its +evidence is explicitly non-price/no-event; execution-grade BARs and every +non-BAR product retain their current event/replay behavior. Missing controls, +gap, stale/finality, identity/source, cursor or strict-current failure remains +terminal. Regression covers quiet-final-BAR success, missing controls and +stale-current rejection, alongside ordinary event replay. Then one final +immutable client rebuild and one fresh 299-product C2 receipt are allowed; no +reader/runtime role is recreated. + +**C2 quiet final-BAR source gate (`PASS / FINAL DISPOSABLE RECEIPT REQUIRED`, +2026-09-03).** The receipt primitive now admits no-event handling only for a +durable, non-execution historical `BAR`: both replica sessions must emit +`REPLAYING` and `LIVE`, and each must immediately pass a strict current, +final, original-requirement V2 snapshot. The resulting handoff is recorded as +`CURRENT_FINAL_BAR_OBSERVED_NO_CURSOR`, explicitly not as a durable event +replay or executable price. `TRADE`, `QUOTE`, `BOOK_*`, execution BAR and every +ordinary BAR event path are unchanged. The closed cases are: missing signed +controls before any current read, and a failed/stale current read after valid +controls. No timeout, freshness, finality, source, identity, gap or policy was +relaxed. + +Evidence from the source-mounted disposable image +`qdl-v2-python:2.0.6-8620303`: historical-BAR focused suite passed **8/8** in +`0.068s`; receipt harness plus identity-pacing suite passed **56/56** in +`1.635s`; the full C2-relevant acceptance set +(`test_phase103_consumer_acceptance`, receipt harness, handoff, identity and +five-liquid handoff) passed **96/96** in `6.390s`. All were run network-less, +read-only, UID/GID `10001`, with dropped Linux capabilities and a tmpfs `/tmp`. +`git diff --check` and syntax compilation passed. The image does not include +`ruff`, so no lint result is claimed; repository CI remains the formatting/lint +authority. No runtime role, provider, Kafka, Redis, SQLite, V1, Trading System, +alpha, Gateway/Risk, order path or runtime identity changed. Next exact action: +commit this source slice, build one immutable final C2 client from it, run one +fresh quota-aware C2 receipt, then remove the two stopped disposable C2 +containers and any non-retained client image after compact evidence is retained. + +**C2 stream-handoff closure (`IN PROGRESS / NARROW SOURCE FIX`, 2026-09-03).** +The final disposable C2 client reached the real V2 read plane and failed +closed at `alpha.binance.paper.stable / BINANCE.USDM.PERPETUAL.DOGE-USDT / +BAR 4h`: the quiet historical-bar path observed repeated `SNAPSHOT_REPLACED` +controls rather than the required signed `REPLAYING -> LIVE` sequence. This is +not a data freshness or provider failure. Read-only inspection proves +`query_v2_1`, `query_v2_2`, `stream_v2_active`, and `stream_v2_passive` all +open the same durable `canonical-cache.sqlite3` generation +`99704a2e9486486e8822156550d1d2e3`, use the same cursor key id and config +revision, and have no restart or OOM event. The active stream target is +currently fenced by the scoped lease; the SDK correctly moves to the peer, but +the peer rejects the query-issued handoff cursor and the SDK keeps replacing +the snapshot. That is a real REST-query-to-gRPC-stream contract failure and +must remain fail-closed. + +Approved scope is deliberately narrow: trace the exact cursor rejection with +a disposable, read-only diagnostic; correct only the shared SDK/stream handoff +or its cursor issuance if the contract is wrong; add deterministic regressions +for active-to-peer failover, fresh query cursor acceptance, expired cursor +replacement, and no cross-instrument/consumer mixing. No timeout/SLA +relaxation, synthetic event, provider change, runtime role recreate, Kafka, +Redis, SQLite, V1, Trading System, alpha, Gateway/Risk or order-path mutation +is allowed. Exit requires an immutable client image without a source mount, +the relevant offline matrix, and one fresh C2 no-order receipt. If the real +diagnostic instead proves a live runtime misconfiguration, source changes stop +and a separate exact runtime packet is required. + +**C2 cursor rejection root cause (`CONFIRMED / HARNESS-ONLY REPAIR`, +2026-09-03).** The disposable mTLS/JWT diagnostic retained only cursor +metadata and proved the issued token is correctly signed, consumer-scoped, +generation-bound and unexpired. The DOGE 4h durable partition currently spans +logical offsets `1..6572`; the artificial one-prior-bar C2 seed carried +watermark `1000`, so the active peer correctly returned `GATEWAY_FENCED` and +the lease-owning peer correctly returned `CURSOR_EXPIRED: replay backlog +exceeds the bounded gateway window`. Repeating the deliberately historical +query merely issued another valid-but-too-old cursor, producing the observed +`SNAPSHOT_REPLACED` loop. The retained diagnostic is +`c2-quiet-final-bar-20260903T005238Z/diagnostic-cursor-handoff-20260903T013035Z/diagnostic.json` +with SHA-256 `3b29bdcab87c8d21dad9dfc1bcaab4a3ef6213a701bb1f59624a39693933539f`; +it made zero provider connections and zero order actions. + +The correction is C2 harness-only: a non-execution durable BAR will use the +normal, current governed warmup boundary for signed `REPLAYING -> LIVE` and +cross-replica reopen. A quiet BAR still needs strict final/current read-back on +both replicas and is recorded as no-event handoff evidence. C2 will no longer +manufacture a stale time-range cursor merely to force replay beyond the +server's deliberate bound. This preserves the real SDK behavior for an +expired consumer cursor: it remains a mandatory fresh snapshot rebuild. Tests +must prove current-boundary controls, quiet final BAR read-back, execution BAR +exclusion, and that an actual expired cursor still fails/rebuilds rather than +being silently accepted. + +**C2 current-boundary source gate (`PASS / IMMUTABLE CLIENT BUILD REQUIRED`, +2026-09-03).** The acceptance harness now uses the real current governed BAR +warmup as its stream handoff boundary. Non-execution durable BARs may be quiet, +but only after the current signed cursor emits `REPLAYING` and `LIVE`, and only +after strict current/final V2 read-back on both query replicas. Execution BAR, +quote/trade/book behavior, `CURSOR_EXPIRED` semantics, replay limit and every +runtime service stay unchanged. A source-mounted, read-only mTLS/JWT probe +against the live V2 plane confirmed DOGE 4h current cursor watermark `6572` +through active-to-peer failover on both query replicas, yielding +`REPLAYING`, `LIVE`, then the expected quiet timeout; it made zero provider +connections and zero order actions. Compact evidence is +`diagnostic-current-handoff-controls-20260903T013858Z/diagnostic.json` under +the same C2 evidence root. + +The focused receipt harness passed **41/41**. The expanded offline, +network-disabled, read-only UID/GID-`10001`, capability-dropped matrix passed +**130/130** in `20.530s`: Phase-10.3 acceptance/receipt, Phase-10.5 consumer, +fallback, handoff and identity paths, shared stream SDK, and workload TLS +security. The gRPC test-server `GOAWAY` diagnostic is expected teardown output; +the test exit was successful. `git diff --check` passed. No V2 runtime role, +provider, Kafka, Redis, SQLite, V1, Trading System, alpha, Gateway/Risk or +order path changed. Next: commit this coherent source slice, build one +immutable disposable C2 client image, run the same no-source-mount matrix, and +run exactly one fresh C2 299-product/four-identity/300-second no-order receipt. + +**C2 current-boundary immutable client (`PASS / ONE REAL RECEIPT IN +PROGRESS`, 2026-09-03).** Commit `3d73891` is sealed as +`qdl-v2-python:2.0.8-3d73891@sha256:674c29164035d8b22a83ce5f10153d47b334ae50293adfb6491e82f62a2aea14`. +OCI `revision` and `version` labels are exactly `3d73891` and +`2.0.8-3d73891`. Its image-contained, no-source-mount regression ran +network-disabled, read-only as UID/GID `10001` with all capabilities dropped, +`no-new-privileges` and a tmpfs `/tmp`; it passed **130/130** in `21.174s`. +Coverage is the C2 consumer/receipt/fallback/handoff/identity suite, shared +stream SDK and workload TLS transport security. This disposable client has no +Docker socket, provider, Gateway/Risk, order, signal, sizing, Kafka, Redis or +SQLite authority. + +The approved next action is exactly one fresh four-identity, 299-product, +300-second C2 receipt using this image and a new evidence namespace. It may +read V2 query/stream and governed V1 fallback only. V2 reader/runtime roles, +V1, Kafka, Redis, SQLite, Trading System, alpha and the order path remain +unchanged. On terminal result, retain only compact hashes/metrics and remove +the disposable C2 container; test-only images are inventoried for exact +post-closure cleanup. + +**C2 current-boundary receipt (`FAIL-CLOSED / RAW READER DIAGNOSIS`, +2026-09-03).** The one authorized `3d73891` receipt exited `1` after roughly +13 minutes; it was not OOM-killed (`137.4 MiB / 512 MiB` while active), and +its post-bootstrap client was UID `10001`, `NoNewPrivs=1`, with effective, +permitted, inheritable and ambient capabilities all zero. It made no receipt, +provider, Gateway/Risk, order, signal, sizing or durable-state mutation. The +first terminal route was `alpha.binance.paper.stable / +BINANCE.USDM.PERPETUAL.BTC-USDT / BAR 1d`: C2 asks the intentionally bounded +but exact public warmup horizon of `700` rows and the SDK rejected the V2 +response because its returned BAR count differed from that horizon. The +failure digest is `36a40d4c1ff413940e53eb48985cc27d4136eab4c3d3018becc6297a32dda51a` +under `releases/2.0.8-3d73891/c2-current-boundary-20260903T014559Z`. + +This is a valid fail-closed data-contract result, not a reason to reduce the +requested horizon or accept a partial response. Next is one scoped read-only +mTLS/JWT diagnostic against both V2 query replicas that records only count, +coverage, finality/open-time bounds, quality and hashed cursor metadata for +this exact governed requirement. No runtime role is recreated until that +evidence distinguishes missing materialized history from an incorrect reader +window/provider projection. + +**C2 durable-history closure (`IN PROGRESS / SHARED BAR-EDGE REPAIR`, +2026-09-03).** The two-replica raw V2 diagnostic now confirms the reader is +correctly fail-closed: `alpha.binance.paper.stable / BTC-USDT / BAR 1d` +requested `700` governed rows and both replicas returned `409 PARTIAL_RESULT` +with no accepted partial payload. The compact read-only evidence is +`releases/2.0.8-3d73891/btc-1d-raw-reader-20260903T020411Z/diagnostic.json` +with SHA-256 `ec2ebd9da90078def19e134d643fc5887966b646a03250f5a8fb12c12760b146`. +Direct durable-cache inspection shows only `95` Binance BTC `1d` BAR rows, +whereas the interval-aware truthful retention ceiling and C2 requirement are +both at least `700`. This is not a reader fallback defect and it must not be +papered over by lowering the warmup, changing `SNAPSHOT_AND_REPLAY` to a +pass-through source policy, weakening finality, synthesizing rows, or silently +returning a partial response. + +The shared defect is that the stable multi-venue BAR edge treats the presence +of a persisted `last_open_ms` checkpoint as proof that durable history remains +complete. A cache generation can retain a current checkpoint while historical +rows were never materialized or were compacted/rebuilt. The repair scope is +one provider-neutral edge validation: on bootstrap, verify bounded durable +history coverage for every enabled Binance/OKX BAR binding against the +configured interval-aware warmup requirement; for a deficient binding, fetch +only the real authenticated provider window, validate calendar/finality and +publish only still-missing rows through the existing raw Kafka -> Rust +canonical -> projector path. It must not delete/flush cache or Redis, reset +Kafka offsets/topology, alter V1, touch Trading System/alpha/order paths, or +accept a partial reader result. Existing targeted-repair primitives remain the +idempotent publisher; no symbol-specific service, image, or worker is added. + +Before the code/runtime slice, exact disposable C2 diagnostic containers and +unreferenced Data Layer test images/build artifacts are inventoried and removed +under the approved cleanup scope to recover the workspace required for SQLite +read-only preflight. Active V2 roles, V1 fallback, Kafka, Redis, projectors, +volumes, runtime/TLS state, and the current C2 image until its closure result +are retained. Pre-clean filesystem state is `/dev/root 290G, used 276G, +available 15G (96%)`; post-clean measurements and exact retention/removal set +are recorded below before any durable repair is attempted. + +**C2 durable-history cleanup and source gate (`PASS / ONE SHARED RUNTIME ROLE +PENDING`, 2026-09-03).** Exact cleanup recovered workspace without touching a +running role, V1, Kafka, Redis, volume, runtime/TLS state, SQLite file or +rollback image: removed six terminal disposable C2/diagnostic containers and +the unreferenced client images `2.0.4-7cd0265`, `2.0.5-a497ab0`, +`2.0.6-8620303`, `2.0.7-5ba4ff1`, plus three unreferenced Rust builder images +(`authority-log-test`, `ready-refresh-test`, `depthupdate-test`). The retained +runtime set is V1 `qdl-v1-fallback:v1.2.4-2b0dcf7`, active query/stream +`c13c...`, edge `2ab6...`, Rust core `c63d...`, projectors `b335...`, current +ingestors, Kafka/Redis and the current disposable client +`qdl-v2-python:2.0.8-3d73891`. Disk changed from `276G used / 15G available` +to `270G used / 21G available`; Docker image footprint changed from `101.1G` +to `94.13G`. No service restarted during cleanup. + +The corrected all-binding read-only preflight uses the actual catalog +`source_id` partition key, not a guessed binding name: `139/140` BAR bindings +meet the current interval-aware `1000`-row warmup target. The sole deficit is +`binance-usdm-btcusdt-bar-1d`, `95/1000`, short `905`; this verifies the repair +is bounded and prevents a broad provider replay. The shared edge now validates +the exact durable calendar window behind every restored checkpoint. It removes +only a deficient binding watermark so the existing bootstrap publishes only +provider-confirmed missing final rows through normal raw Kafka/Rust/projector +flow; all healthy bindings retain their checkpoint and are not re-fetched. + +Source gates passed with `qdl-v2-python:2.0.8-3d73891`, source mounted +read-only, network disabled, UID/GID `10001`, capability drop, no-new-privs +and tmpfs `/tmp`: full BAR bootstrap/recovery/cache-generation suite +`37/37` in `12.072s`, then C2 consumer/receipt/fallback/handoff/identity, +shared SDK and workload-TLS matrix `130/130` in `21.123s`. The expected gRPC +GOAWAY was teardown-only; both test exits were successful. Next is to seal one +immutable Python image from this commit, run its no-source-mount regression, +then rolling-recreate **only** `binance_bar_edge`. Its startup may make the +normal, provider-authentic final-BAR writes for this one verified BTC `1d` +deficit. Rollback is the current `2ab6...` image/runtime mount; no cache +deletion, offset reset, topology change or consumer/order mutation is allowed. + +**C2 checkpoint-validation startup bound (`IN PROGRESS / SOURCE-ONLY +PERFORMANCE CORRECTION`, 2026-09-03).** The first one-role start on the new +image exposed an implementation cost without weakening any correctness gate: +checkpoint validation called the physical-tail reader for each of 140 healthy +bindings and eagerly decoded all `10,064` retained payloads per partition even +when the bounded expected warmup set was already complete. The role remained +fail-closed/running with no OOM, provider publish, Kafka or cache mutation, but +startup is not acceptable at that complexity. The narrow correction retains the +same physical-tail query, identity/finality checks and missing-open semantics; +it iterates the SQLite cursor in bounded batches and stops as soon as every +expected open has been proven. A deficient binding still scans its bounded tail +and remains marked for normal authenticated repair. Required regression proves +early completion, late-tail visibility, mismatch fail-closed behavior and the +existing 140-binding checkpoint contracts. Then seal one replacement shared +image and recreate only the same edge again with the already-rendered exact +runtime mount. No provider/service/data-plane policy changes are permitted. + +**C2 durable-history one-role runtime packet (`APPROVED / READY`, 2026-09-03).** +The exact replacement coordinate is +`qdl-v2-python:2.0.9-ce530ef@sha256:344ba6abce45e39c0dbe9834df36d06154a18da975e5bb417d725f18cdf4d829`. +The packet renders the same Compose project `qdl_v2_stable_candidate` with the +same env file, `docker-compose.v2-stable.yml`, current C2 override and sealed +BAR projection override, plus a disposable image-only override for +`binance_bar_edge`. It recreates that role with `--no-deps --force-recreate`; +its three existing mounts stay exactly `/runtime:ro` from +`session-liveness-43cdbe3-20260829T162719Z/runtime`, the existing +`stable_state` volume read-write and existing `stable_tls` volume read-only. +The current rollback coordinate is the explicitly sealed +`qdl-v2-python@sha256:2ab6a946e5ca59e234063fb661a7a7738795dab124e371b5579db31febfd42e0` +with the same compose inputs and mounts. Excluded: every other role, V1, +Kafka topology/offsets, Redis, SQLite deletion/reset, Trading System, alpha +and order path. Post-roll requirements are no restart/OOM, exactly one +provider-authentic BTC `1d` history repair bounded by the observed 905-row +deficit, projector convergence, all-binding preflight and a fresh C2 receipt. + +**C2 durable-history first runtime result (`CONVERGED / EXACT GAP AUDIT +EXPANDED`, 2026-09-03).** The first image-only recreate failed closed before +provider access because the inherited Compose env file mounted an older +`phase103` runtime directory while the running edge's inspected mount was the +sealed `session-liveness-43cdbe3-20260829T162719Z/runtime` directory. No row, +checkpoint or external action was emitted by that failed attempt. The packet +was corrected by rendering and preserving the inspected mount exactly, then +recreating only `binance_bar_edge` again on `2.0.9-ce530ef`; it is running, +`restart=0`, `OOMKilled=false`. + +The exact open-time validation correctly proved that count-only preflight was +insufficient: it found 38 affected bindings (not merely BTC `1d`) with 22,351 +missing authentic opens in their current windows. The edge fetched only their +provider-confirmed final windows and published `22,355` normal final-BAR +events through raw Kafka -> Rust canonical -> existing projectors. It included +BTC `1d` (`999` rows at the current provider window, one already durable) and +hidden gaps across Binance/OKX short/mid intervals; no record was fabricated, +deleted, or written directly to SQLite, and no V1/Trading System/alpha/order +route was touched. The updated checkpoint reached all 140 binding watermarks +and the role resumed normal close ACKs for both venues. The small difference +between detected missing and ACK totals is expected overlap with concurrent +current final-BAR ACKs; post-projection exact coverage preflight remains +mandatory before C2. + +The source-only startup-bound correction is now implemented: physical-tail +coverage keeps the same bounded `10,064` SQL window and all identity/finality +checks but consumes cursor batches of `256` and returns as soon as the expected +opens are proven. The regression injects an unreadable payload after a complete +expected set, proving it is never decoded; missing/mismatched tails remain +fail-closed. Offline source-mounted suite passed `167/167` in `32.666s` +(BAR history/cache recovery plus C2/SDK/fallback/handoff/identity/TLS), with +syntax and diff checks passing. Next is a single coherent commit, one new +immutable shared image, its no-source-mount regression, then one final +one-role recreate against the preserved sealed runtime mount. It must observe +zero historical repair requirement after projector convergence; any new normal +closed BAR at a real boundary remains allowed. + +**C2 checkpoint-history closure packet (`IN PROGRESS / FINAL NO-ORDER +RECEIPT NEXT`, 2026-09-03).** Commit `82b4830` +(`perf(bar-edge): bound checkpoint coverage reads`) preserves the exact +provider-neutral durability contract and improves only healthy-start cost: its +physical SQLite tail cursor reads at most `256` rows at a time and exits once +the requested calendar opens are proven. It does not lower the `10,064` tail, +relax identity/finality checks, alter a deficient binding's repair path, or +introduce a cache shortcut. The immutable image is +`qdl-v2-python:2.0.10-82b4830@sha256:74617d3ffc79dc539d1d25dfd025546728d4ceeb5c93ff57c29e919a4d6fef74`. +The image-contained, no-source-mount, network-disabled, read-only UID/GID +`10001`, capability-dropped matrix passed `167/167` in `34.350s`. + +The approved one-role rollout rendered only `binance_bar_edge` with that +image, the existing `stable_state` and read-only `stable_tls` volumes, and the +inspected sealed runtime directory +`session-liveness-43cdbe3-20260829T162719Z/runtime`. All other V2 roles, V1, +Kafka topology/offsets, Redis, SQLite deletion/reset, Trading System, alpha +and order paths remain excluded. The role is `running`, `restart=0`, +`OOMKilled=false`. Exact calendar-window validation correctly found 18 newly +missing real final BARs across 14 Binance/OKX `5m/15m/30m` bindings; they were +published only through the normal provider -> raw Kafka -> Rust canonical -> +projector flow and bootstrap completed `bindings=140 rows=18`. This reflects +normal window advance after the earlier checkpoint, not a count-only bypass or +synthetic write. + +The sole next action is one fresh Phase-10.5 C2 four-identity, 299-product, +true-300-second no-order receipt from the immutable `2.0.10-82b4830` image. +It uses only V2 query/stream and its explicitly governed local V1 fallback +drill; it receives no Docker socket, provider credential, Gateway/Risk/order, +signal, sizing, Kafka, Redis or SQLite authority. The disposable client starts +as root only to copy declared mTLS/JWT files into tmpfs, then drops to UID/GID +`10001` with `NoNewPrivs=1` and all capabilities cleared. Its fresh evidence +namespace retains only bounded receipt/security/error hashes. A nonzero result +remains fail-closed; after the terminal result, remove only the named +disposable client and obsolete C2 test images/temporary override once they are +unreferenced. No retry may weaken history, freshness, gap, identity, cursor or +fallback policy. + +**C2 durable-history receipt result (`FAIL-CLOSED / READ-ONLY L2 SESSION +DIAGNOSIS`, 2026-09-03).** The one disposable client ran from the immutable +`2.0.10-82b4830` image, was removed automatically on exit, and retained only +the bounded stderr hash +`1953a7cbd83f77613e47acdcb2ee09c50a6e70e89f2637ff2bf64020ba3837d5` in +`releases/2.0.10-82b4830/c2-durable-history-20260903T024207Z`. It started as +root solely for tmpfs identity copy, then recorded UID `10001`, zero effective, +permitted, inherited and ambient capabilities, and `NoNewPrivs=1`. It made no +provider, Gateway/Risk, order, signal, sizing, Kafka, Redis or SQLite action. +The terminal requirement is +`trading-system.paper.stable / OKX.SWAP.PERPETUAL.BTC-USDT / BOOK_DELTA`: +the strict warmup returned typed freshness rejection and its bounded retry +correctly refused to accept a session lacking live-provider evidence. This is +not a BAR repair regression and is not eligible for an SLA relaxation, +synthetic BOOK_DELTA, fallback substitution or retry-by-luck. + +Next scope is read-only only: query typed status/view lineage for the exact +OKX BTC `BOOK_DELTA` identity from both V2 replicas and inspect the shared +Rust/book projection health. If it is a genuine provider disconnect, keep C2 +blocked and record the operational condition; if both replicas show a +healthy provider session but an incorrect shared projection/liveness state, +repair only that shared Rust/provider-lineage path with regression evidence, +then rebuild the immutable client and run one new C2 receipt. V1, Kafka, +Redis, SQLite deletion/reset, Trading System, alpha and the order path remain +out of scope. + +**C2 L2 typed-status diagnosis (`PASS / HARNESS-ONLY TRANSIENT-RECOVERY +REPAIR`, 2026-09-03).** A fresh bounded, read-only probe used the same Trading +System paper identity against both V2 replicas for the exact failed identity. +Both return `state=LIVE`, `complete=true`, `gap_open=false`, +`execution_eligible=true`, `event_recency_state=LIVE`, +`provider_session_state=LIVE`, session liveness `517ms`/`559ms`, and a strict +snapshot succeeds. The probe had no provider/order/durable-state authority; +its output is `releases/2.0.10-82b4830/l2-status-20260903T025900Z/status.json`. +This rules out a persistent OKX admission, symbol binding, sequence, gap, +Rust projection or replica-cross-mix defect. The C2 terminal point instead +caught a genuine short-lived provider/session transition and the current +harness treated the first non-LIVE status as permanently terminal. + +The corrective scope is harness-only and deliberately does not change serving +freshness, reader behavior or retryable data semantics. During a C2 strict +`QUOTE`/`BOOK_DELTA` retry, an identity-matched, complete and gap-free +transitional session (`UNKNOWN`, `STALE` or `DISCONNECTED`) may be re-polled +only until the existing strict deadline; no transitional snapshot is accepted, +and final admission still requires an independently read `LIVE` session within +the declared liveness SLA plus a new strict snapshot. Gapped, incomplete, +identity/policy mismatch, bad status response and deadline expiry remain +terminal. Regression must cover recovery to `LIVE`, persistent disconnect, +gap rejection and no cross-symbol state reuse. The following immutable-client +and one fresh C2 receipt remain the only permitted runtime actions. + +**C2 transient-session source repair (`PASS / IMMUTABLE CLIENT RECEIPT NEXT`, +2026-09-03).** The repair is confined to +`scripts/phase103_consumer_receipt_acceptance.py`: strict C2 retry for only +`QUOTE` and `BOOK_DELTA` now re-polls an exact identity-matched, complete, +gap-free, non-executable reconnect state (`UNKNOWN`, `STALE` or +`DISCONNECTED`) on a bounded `0.5s` cadence inside the already-declared +deadline. It neither changes SDK/query/Rust freshness rules nor admits that +transitional response. The next snapshot must independently pass the existing +fresh/live/session-SLA/complete/gap-free typed contract. `BOOK_SNAPSHOT`, +`TRADE`, policy mismatch, cross-symbol status, open gap and persistent +disconnect retain fail-closed behavior. + +The source-only, network-disabled, read-only UID/GID `10001` regression +evidence is: `test_phase103_consumer_receipt_harness.py` `44/44` in `9.618s`, +`test_phase105_consumer_acceptance.py` `7/7` in `8.844s`, +`test_phase105_fallback_acceptance.py` `7/7` in `6.405s`, and +`test_phase115c_five_liquid_handoff.py` `8/8` in `0.168s`. The new harness +cases prove recover-to-fresh for quote and book delta, persistent disconnect +deadline failure, open-gap rejection and no cross-symbol status reuse. No +runtime role, provider, Kafka, Redis, SQLite, V1, Trading System, alpha or +order path was changed. Next permitted action: commit this source slice, build +one immutable disposable C2 client image and run exactly one fresh C2 receipt. + +**C2 transient-session immutable receipt (`FAIL-CLOSED / BAR GAP +DIAGNOSIS`, 2026-09-03).** Commit `10bb4c6` was sealed as the one disposable +client image `qdl-v2-python:2.0.11-10bb4c6@sha256:d3254eaddf769547fc017f63e066ddf6db9f70404f40a4b656cd329647125227`; +OCI revision/version are exact and the immutable, no-source-mount, +network-disabled C2 matrix passed `66/66` in `17.076s`. The one authorized +four-identity no-order client then ran about 14 minutes at roughly `116-134MiB` +of its `512MiB` bound, self-removed, and produced no order, signal, sizing or +durable-store action. It did not reach the observation window: the first +terminal condition is `alpha.binance.paper.stable / BINANCE.USDM.PERPETUAL. +BTC-USDT / BAR 1h`, whose strict warmup correctly rejected the typed V2 +response with `required feed has an unresolved sequence gap`. The compact +stderr is `3,948` bytes and is retained only in +`releases/2.0.11-10bb4c6/c2-transient-session-20260903T031000Z`. + +This proves the transient-session retry did not weaken read admission. Next +scope is read-only two-replica status/warmup lineage for that exact Binance +`BTC 1h BAR` identity, then targeted shared bar-edge/Rust/projector repair only +if a real materialized gap is confirmed. No retry C2 is permitted until that +truthful diagnosis and its regression exit are recorded; V1, Kafka topology or +offsets, Redis, SQLite reset/deletion, Trading System, alpha and order paths +remain excluded. + +**C2 BTC 1h BAR lineage (`CONFIRMED / SHARED CACHE-GENERATION REPAIR`, +2026-09-03).** Read-only inspection of the exact canonical partition +`a953e16e-7138-5562-b5e8-c337a44d0b65/bar/binance-usdm-btcusdt-bar-1h-primary-v2` +proved a real 24-open hourly hole, not a C2, query or SDK false positive: +the bounded durable tail had `703` distinct FINAL opens, no duplicate opens, +and `24` missing expected hourly opens. The typed latest status can still be +`LIVE/complete/gap_open=false` because it describes the current provider +session, whereas governed warmup correctly refuses the historical hole. No +synthetic event, fallback substitution, SLA relaxation or retry was used. + +Root cause is a shared lifecycle gap: after a canonical SQLite generation +rebuild, a continuously running BAR edge can retain its old checkpoint +watermarks because ordinary recurring final-BAR polling does not currently +compare the cache generation before deciding that no binding is due. The +narrow source repair is provider-neutral and applies to every Binance/OKX +BAR binding: detect a changed canonical-cache identity before the next +bootstrap/poll, clear only the edge's in-memory watermark/retry state, issue +a new source connection generation, persist the rebased V4 checkpoint, then +run the existing bounded provider-authentic bootstrap path. It does not alter +Kafka topology/offsets, Redis, SQLite deletion, query policy, Rust domain +logic, V1, Trading System, alpha or any order path. Tests must prove one +generation change triggers one truthful re-bootstrap, old watermarks are never +reused, same-generation polling is a no-op, and missing/invalid cache identity +continues fail-closed. Only after this source gate, an immutable image and a +bounded V2-only recovery can repair the observed cache before exactly one new +C2 receipt. + +**C2 live cache-generation source repair (`PASS / RUNTIME RECOVERY NEXT`, +2026-09-03).** `StableBinanceBarEdge` now performs one inexpensive +canonical-cache identity read at the beginning of its existing bootstrap path. +If and only if the identity changed, it clears only that edge's in-memory +watermark/retry maps, issues a new provider connection generation, atomically +persists a V4 checkpoint bound to the new cache identity, then enters the +existing bounded authenticated Binance/OKX history bootstrap. Equal identities +are a no-op; an unavailable/invalid identity remains fail-closed; a generation +change during a publish still fails closed and is repaired on the next loop. +No per-symbol worker, service, provider bypass, synthetic data, reader policy +or public contract was added. + +The source-mounted, network-disabled, read-only UID/GID-`10001`, +capability-dropped test matrix passed **105/105 in 24.176s**: +all stable BAR history/recovery/cache-generation tests plus Phase-10.3 receipt, +Phase-10.5 acceptance/fallback and Phase-11.5-C handoff suites. New regression +proves a live cache generation change clears old watermarks/retries, increments +the source generation, persists the rebased checkpoint, reboots every governed +Binance/OKX BAR binding through the bounded bootstrap path exactly once, and +does not rebase an unchanged cache; unreadable identity fails before mutation. +The isolated test had no provider, Kafka, Redis, SQLite, V1, Trading System, +alpha, Gateway/Risk or order-path authority. Next is a bounded V2-only runtime +recovery from the retained state/volumes, real-provider repair of the confirmed +hole, projector convergence verification, then exactly one fresh C2 receipt. + +**C2 V2 runtime recovery packet (`APPROVED / PRE-FLIGHT PASS`, 2026-09-03).** +Read-only Docker inventory found no remaining V2 containers or local V2 images; +the external cause is not asserted. V1 remains live as +`data_layer_service@qdl-v1-fallback:v1.2.4-2b0dcf7`; the existing V2 Compose +volumes (`kafka1_data`, `kafka2_data`, `kafka3_data`, `stable_state`, +`stable_tls`, `stable_authority_db`) and external `executor_network` remain +present. No V2 volume, Kafka offset, Redis key, SQLite file, V1 service, +Trading System, alpha or order path will be reset, deleted or changed. + +Recovery uses the canonical existing Compose project +`qdl_v2_stable_candidate`, the sealed runtime directory +`session-liveness-43cdbe3-20260829T162719Z/runtime`, and its preserved private +environment. Only a mode-`0600`, payload-free selector overlay changes the +Python/Rust image coordinates to the newly built immutable shared images from +commit `8ba4165`: Python +`qdl-v2-python:2.0.12-8ba4165@sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d` +and Rust +`qdl-v2-rust:2.0.12-8ba4165@sha256:d86f0e832ba945d302fd3f782e26fd41c5b08709a80f6de16bdd36af5ed86983`. +Both OCI labels are revision `8ba4165`, release `2.0.12-8ba4165`, and run as +non-root. The Python image-contained, no-source-mount, network-disabled +matrix passed **105/105 in 24.087s**. Compose render passed and resolves the +existing 21-role topology only; no per-symbol/container topology is created. + +The allowed runtime effect is normal real-provider V2 market-data traffic. +On bar-edge startup, the existing V4 checkpoint validation must detect the +confirmed missing BTC 1h window, re-bootstrap only that deficient binding +through raw Kafka -> Rust canonical -> existing projector/cache, and leave all +other covered bindings untouched. Rollback is stopping only the restored V2 +project; V1 remains available. Post-start gates are all role health/no OOM, +projector catch-up, exact coverage `0` for the failed binding, then one and +only one fresh C2 four-identity no-order receipt. + +**C2 V2 recovery first-start correction (`FAIL-CLOSED / APPROVED CONTINUATION`, +2026-09-03).** The restored base Compose topology started only its existing +V2 roles and did not touch V1, Trading System, alpha, order state, Kafka +topology or durable volumes. It correctly stopped rather than accepting an +ambiguous runtime: `binance_bar_edge` rejected the image-default catalog while +the retained environment still named an obsolete r14 checkpoint; all three +projectors rejected a new ephemeral Redis identity while their durable spool +is non-empty (`ProjectionCacheMismatch`). These are configuration/recovery +guards, not provider data failures. + +Read-only checkpoint inventory corrects the initial r14 hypothesis before any +write: r14 is a legacy `catalog=7/acquisition=14` state, while the only strict +match for the current 140 governed BAR bindings is the existing V4 checkpoint +`phase54-alpha-demand-5edbc8c.json`, paired with sealed +`runtime/phase54-alpha-demand-5edbc8c/{catalog,acquisition}.yaml`, revisions +`8/16`, and `QDL_STABLE_BAR_WARMUP_ROWS=1000`. The continuation therefore +reuses that already sealed projection through one external, mode-`0600`, +non-secret Compose override for the existing `binance_bar_edge` only. It pins +those two paths, the matching V4 state path, and the matching warmup value; it +creates no service, binding, worker, image or public contract. A private +composed environment will bind that override plus the immutable `8ba4165` +image selectors. Before apply it must pass Compose render and the existing +projection-rebuild tool dry-run. The approved apply remains exactly the +governed V2-only cache rebuild: stop the seven V2 cache users, remove only +canonical SQLite `sqlite3/-wal/-shm`, FLUSHDB only `stable_redis`, reset only +`stable-projector-v1` on `md.canonical.v2` to 900 seconds, then start stream +-> projector -> query. Afterward the edge rehydrates provider-authentic BAR +history through the existing Kafka/Rust/projector path and C2 remains blocked +until the exact coverage and fresh receipt gates pass. V1, Trading System, +alpha, order path, other Redis DBs, Kafka topology/other offsets and all +volumes remain excluded. + +**C2 recovery composed-config preflight (`PASS / APPLY AUTHORIZED`, +2026-09-03).** The private selector and the one `binance_bar_edge` override +rendered cleanly with the immutable Python image +`sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`, +sealed `phase54-alpha-demand-5edbc8c` catalog/acquisition paths, matching V4 +checkpoint, `warmup_rows=1000`, and the existing session-liveness runtime +mount. The governed rebuild tool's dry-run names only the approved seven cache +users, three SQLite files, `stable_redis` DB, `stable-projector-v1` group and +15-minute canonical replay. Read-only preflight also proves all three +projectors share that same immutable Python image and that its embedded +catalog SHA-256 equals the source catalog SHA-256 +`c2fe0fe5326856ffb504fc4c2251ac77de9bc315e743248c543b394d8df18d3b`. +No runtime mutation occurred in these preflights. The approved rebuild may now +run exactly once; if it fails, it stops fail-closed without widening scope. + +**C2 recovery cache-rebuild result (`FAIL-CLOSED / THROUGHPUT DIAGNOSIS`, +2026-09-03).** The governed apply executed its exact V2-only scope: stream +then projector replicas started and the isolated Redis/cache identity guard was +cleared through the documented SQLite/Redis/group recovery route. All three +projectors remained running with `OOMKilled=false` and restart count `0`; no +V1, Trading System, alpha, order path or topology mutation occurred. The +script deliberately did not start query because its fixed 900-second recovery +deadline expired before the live canonical group met its `<=250`-record lag +gate (`last_lag=12,199`, six partitions). This is an honest incomplete +recovery, not success and not a reason to reset/replay again. The remaining +work is read-only throughput/lag diagnosis of the already-running projector +group, then a narrow runbook correction only if the observed rate proves that +the fixed deadline is invalid for the approved bounded live replay. Query/edge +and C2 remain blocked; no timeout/SLA/finality policy may be relaxed and no +second cache rebuild is authorized by this result. + +**C2 recovery live-lag gate correction (`APPROVED / SOURCE-ONLY`, +2026-09-03).** Five read-only Kafka samples after the failed script prove the +projector group is caught up to a bounded live tail rather than stalled: +observed totals were `344, 177, 432, 359, 394` across all six canonical +partitions; no individual partition exceeded `236`. The three projectors +remained `running`, `OOMKilled=false`, restart `0`, at roughly `84-106 MiB` +of their `768 MiB` limits. The previous `total <=250` gate is therefore an +invalid aggregate criterion for a six-partition live stream: it can reject a +healthy current tail even when every partition remains within the old +per-partition-sized bound. + +The narrow source correction retains the original `250` record bound per +partition, adds an explicit `500` aggregate cap, parses/records both values, +and requires three consecutive samples satisfying both before query starts. +It also raises only the recovery-command default observation budget from 900 +to 1200 seconds so one bounded real replay can complete under the measured +live ingress; it does not alter V2 endpoint freshness, finality, sequence, +cursor, fallback, data-retention, consumer or runtime capacity policy. Unit +tests must reject an over-limit single partition even if the aggregate is low, +reject an aggregate over 500, preserve the exact six-partition requirement, +and record both limits in the dry-run plan. Existing active projectors are not +reset or restarted for this source slice. After the source gate, the already +recovered runtime may continue from its current offsets: wait for the new +two-dimensional gate, start only the existing query replicas, verify their +typed readiness, then recreate the one shared BAR edge with the sealed V4 +projection. C2 remains blocked until all later data-quality evidence passes. + +**C2 recovery live-lag source gate (`PASS / RUNTIME RESUME READY`, +2026-09-03).** `rebuild_v2_stable_projection_cache.py` now records and enforces +the two-dimensional gate (`total <=500`, each partition `<=250`, all six +partitions, three consecutive samples) and uses a 1200-second default +observation budget. The parser rejects repeated partition lines rather than +silently double-counting them; the dry-run plan now exposes both bounds. +Focused isolated regression passed **16/16** with the immutable +`qdl-v2-python:2.0.12-8ba4165` image, source mounted read-only, network +disabled, UID/GID `10001`, capability-free, and an ephemeral noexec `/tmp` +tmpfs solely for `tempfile` fixtures. An initial fully read-only attempt +correctly revealed that those standard-library fixtures need scratch space; +it did not run the affected cases or mutate source/runtime. No V2 role was +recreated for this source gate. The current live projector group may now be +observed under the corrected gate and query may start only after it passes. + +**C2 recovery runtime resume and final receipt (`IN PROGRESS / ONE DISPOSABLE +NO-ORDER CLIENT`, 2026-09-03).** The recovered live projector group subsequently +met the corrected two-dimensional gate in three read-only samples: totals +`113`, `401`, and `114`, with maximum individual partition lags `39`, `177`, +and `41` respectively across all six partitions. Only the existing +`query_v2_1` and `query_v2_2` roles were started after that proof. Only the +existing shared `binance_bar_edge` role was then recreated with the sealed V4 +`phase54-alpha-demand-5edbc8c` projection; its real-provider bootstrap +acknowledged exactly `140` governed Binance USD-M/OKX Swap BAR bindings and +persisted `catalog=8`, `acquisition=16`, `warmup_rows=1000`, `bindings=140`, +`watermarks=140`. The three projectors have since converged to an aggregate +canonical lag of `178`; query replicas report healthy and the edge has no +restart/OOM evidence. This is real provider -> raw Kafka -> Rust canonical -> +projector/cache data only, not a synthetic repair. + +The sole remaining closure gate is exactly one fresh Phase-10.5 C2 +four-identity, 299-product, true-300-second no-order receipt from immutable +`qdl-v2-python:2.0.12-8ba4165`. The disposable client may read the existing V2 +query/stream pair and perform the governed local V1 fallback-return drill only. +It has no Docker socket, provider credentials, Kafka, Redis, SQLite, +Trading-System, alpha, Gateway/Risk, signal, sizing, or order authority. +It mounts selected workload material read-only, copies only required credentials +to an in-memory tmpfs before dropping to UID/GID `10001` with all capabilities +cleared, retains only bounded receipt/error hashes, and self-removes on exit. +Rollback is stopping/removing that named client; V1 and every running service, +volume, offset, runtime/TLS mount and data-plane role are excluded. A nonzero +receipt is terminal fail-closed and will be diagnosed narrowly before any new +runtime mutation. + +**C2 client mount correction (`IN PROGRESS / HARNESS-ONLY`, 2026-09-03).** The +first disposable C2 container self-removed before any V1/V2 request, provider +connection, durable-store access or order action: the rootless Docker mapping +could not traverse the broad host `/v2state` bind mount. Its terminal stderr is +only the bounded bootstrap permission failure. The corrective launch changes +no data-plane code or service: a host `/dev/shm` staging tree copies exactly +one CA file and twelve named mTLS/JWT files required by the four declared +identities, preserving their expected temporary `/v2state` paths. The staging +tree is mode-`0700`, RAM-backed, mounted read-only, and removed by the host +trap when the client exits; no secret enters the evidence namespace. The client +receives no other state file, then performs the existing tmpfs copy and +UID/GID-`10001` capability-free drop before its first request. A minimal +bootstrap-only `DAC_READ_SEARCH`, `SETUID`, `SETGID` set is confined to those +explicit read-only files and is cleared by `setpriv`; no Docker socket or host +mutation is introduced. The retry remains one full C2 observation, not a +data-plane retry, and its prior zero-request exit does not count as an +acceptance result. + +**C2 reader trust configuration repair (`APPROVED / SOURCE AND FOUR-ROLE +ROLLING ONLY`, 2026-09-03).** The second harness attempt reached UID/GID +`10001`, `NoNewPrivs=1` and zero effective capabilities, then stopped at its +first V2 `TRADE` request for `monitoring / BINANCE.USDM.PERPETUAL.BTC-USDT`. +Both query replicas stayed healthy with zero restart/OOM. Public certificate +inspection proves the external monitoring/OKX identities are valid and the +existing query/stream `client-ca-bundle.crt` files already contain their issuer. +The actual fault is narrower: base Compose supplies only +`QDL_STABLE_TLS_CA_FILE`, so the runtime defaults client authentication to the +server CA and ignores the installed additive bundle. + +The source correction makes query/stream use their additive client-CA bundle +as the canonical client-auth authority. `stable_tls_init` deterministically +copies the server CA into a missing bundle for a base deployment, preserving +legacy one-CA startup; when an approved bundle is supplied it remains unchanged. +Focused source tests must prove all four public reader roles name the bundle and +the bootstrap fallback cannot overwrite it. The bounded runtime packet then +rolls only `query_v2_1`, `query_v2_2`, `stream_v2_active`, and +`stream_v2_passive` with existing image, runtime/TLS/state mounts and the +already-tracked C2 overlay. Its `rust_core` stanza is configuration only and is +not recreated. Rollback re-creates those same four roles with the preceding +Compose revision and server-CA client authority. Kafka, Redis, SQLite, V1, +Rust/ingestors/projectors/bar edge, Trading System, alpha and order paths stay +excluded. After the four roles are healthy, one new full C2 receipt starts from +zero; the prior transport failure remains a terminal harness evidence, not a +successful observation. + +**C2 reader trust source gate (`PASS / FOUR-ROLE ROLL NEXT`, 2026-09-03).** +Base `docker-compose.v2-stable.yml` now declares +`QDL_STABLE_TLS_CLIENT_CA_FILE` on exactly the existing two query and two +stream roles. `stable_tls_init` preserves a supplied additive +`client-ca-bundle.crt`; only when it is absent does it seed that file from +the existing server CA, so a one-CA deployment remains backward-compatible. +There is no new image, service, port, volume, credential, authority or +provider behavior. The focused source matrix +`python -m unittest tests.test_phase105_handoff tests.test_phaseb_stable_edge` +ran from the existing immutable `qdl-v2-python:2.0.12-8ba4165` image with +`--network none --read-only`: **66 passed, 1 intentional skip, 16.106s**. +The exact base-plus-C2-overlay Compose render also passed. A first assertion +count typo was corrected before the final pass and before any runtime action. + +The next bounded runtime operation recreates only `query_v2_1`, +`query_v2_2`, `stream_v2_active`, and `stream_v2_passive` with their +current immutable image, current state/TLS/runtime mounts and C2 overlay. +Rollback is the same four roles with a private external override that sets the +preceding server-CA client authority. Kafka, Redis, SQLite, V1, Rust, +ingestors, projector, bar edge, Trading System, alpha and every order path +remain excluded. Only after reader health and exact env verification may one +fresh C2 299-product/four-identity/300-second no-order receipt run. + +**C2 admission-runtime preflight correction (`APPROVED / FIVE-ROLE ROLL`, +2026-09-03).** Read-only inspection of the recovered runtime found +`rust_core` still has `QDL_PROVIDER_ADMISSION_ENABLED=false`; the sealed C2 +overlay already sets it to `true`, but a `--no-deps` reader-only Compose +roll cannot apply an environment change to a running core. Without this exact +private listener, the C2 reference-data products would fail truthfully after +the mTLS repair. This is a recovery-config omission, not a new architecture, +image, route, symbol worker or authority model. + +The bounded packet therefore recreates existing `rust_core` first using its +current immutable Rust image, existing core runtime/TLS/Kafka/Redis mounts and +the already-sealed `QDL_PROVIDER_ADMISSION_ENABLED=true` overlay; it then +recreates only the same four reader roles with their current immutable Python +image. The other two Rust replicas, ingestors, bar edge, projectors, V1, +Kafka topology/offsets, Redis, SQLite, Trading System, alpha and order path +remain excluded. The rollback override explicitly restores this one core to +`false` and the four readers to their preceding server-CA authority, then +recreates only those same five roles. Successful private listener readiness +and all five role health are required before the single fresh C2 receipt. + +**C2 trust/admission runtime roll (`PASS / FRESH RECEIPT RUNNING NEXT`, +2026-09-03).** The bounded packet recreated only the existing `rust_core` +and then `query_v2_1`, `query_v2_2`, `stream_v2_active`, and +`stream_v2_passive`. All five retain their prior immutable image digest; +there was no build, new container class, new port, new volume, Kafka/Redis/ +SQLite operation, V1 operation, Trading System/alpha operation or order-path +mutation. `rust_core` is running with restart `0`, `OOMKilled=false`, +`QDL_PROVIDER_ADMISSION_ENABLED=true`, and emitted +`qdl_provider_admission_started` with its sealed policy hash. Each reader is +`healthy`, restart `0`, `OOMKilled=false`, and has its exact additive +`client-ca-bundle.crt` environment path. Bounded startup logs show normal +Uvicorn readiness only. The explicit runtime rollback override retains the +prior server-CA path for these same four readers and `false` for this same +core; no other role is in either direction. + +The next and only acceptance action is a fresh disposable C2 namespace: four +declared identities, all 299 governed products, real V2 query/stream reads, +one governed V1 fallback-return drill and a true 300-second no-order +observation. It starts from zero, has no Docker socket, provider credential, +Kafka/Redis/SQLite, Trading System, alpha, Gateway/Risk, signal, sizing or +order authority, self-removes on exit, and retains only bounded receipt/error +hashes. Any nonzero outcome remains terminal fail-closed. + +**C2 trust/admission receipt (`FAIL-CLOSED / TYPED TRADE-SESSION +DIAGNOSIS`, 2026-09-03).** The fresh disposable client self-removed after +`17.99s`; its RAM-only credential staging directory was immediately removed. +It reached UID/GID `10001` with `NoNewPrivs=1` and zero effective +capabilities, proving the prior mTLS transport failure is repaired: the query +returned a typed V2 freshness response rather than disconnecting. It made no +order/signal/sizing/Gateway/Risk mutation and retained only its bounded stderr +hash `bbec929b69f04b78fcb0251621906372e7b2763912757c9d9ad3a155cac0cdad`. +The receipt is not successful: the first monitoring product +`BINANCE.USDM.PERPETUAL.BTC-USDT / TRADE` reported +`required data exceeds its freshness policy`, and strict retry exhausted +because its provider session was not `LIVE`. `acceptance.json` is empty; +the reader replicas stayed healthy and no C2 client container remains. + +This proves the reader trust correction, but not V2 consumer readiness. It +does not justify an SLA change or another receipt. The sole next scope is +read-only typed status/lineage for this exact Binance trade identity from both +query replicas and the matching shared ingestor/Rust/projection path: session +state, event recency, gap, watermark and completeness. If the status is +incorrect while the provider session is live, repair only the shared +projection/provider lineage with regression evidence; if the session is truly +down, restore it through the existing shared role and record the real provider +condition. C2 remains blocked until that exact condition is resolved. + +**C2 Binance TRADE typed-status diagnosis (`CONFIRMED / HARNESS-ONLY +SOURCE REPAIR`, 2026-09-03).** A disposable, read-only, mTLS/JWT +monitoring probe queried the exact failed `BINANCE.USDM.PERPETUAL.BTC-USDT / +TRADE` identity from both V2 query replicas. Both returned the same governed +truth: `state=LIVE`, `event_recency_state=LIVE`, `complete=true`, +`gap_open=false`, `execution_eligible=true`, `policy_id=crypto_primary_v2` +and sub-second freshness. The latest indexed canonical event also matches the +currently `LIVE` Binance source session and config generation; all shared +ingestor/core/projector roles remain running, restart `0`, OOM false. + +The observed `provider_session_state=NOT_APPLICABLE` is correct, not an +outage: the affected monitoring TRADE manifest deliberately declares no +`max_session_liveness_ms`. C2's retry branch nevertheless treated every +TRADE stale-snapshot race as requiring a `LIVE` session and rejected the +subsequent fresh/executable typed status. The correction is confined to the +acceptance harness: a TRADE without a declared session SLA may re-read only +when its status is already identity-matched, `LIVE`, current, complete, +gap-free and execution-eligible with `NOT_APPLICABLE` session fields. It +never admits a stale price, quiet trade, disconnected provider or missing +session as success; the next snapshot must still pass ordinary strict V2 +freshness. Regression must cover the permitted fresh race and rejected +quiet/disconnected cases. No provider, runtime role, image, manifest, +freshness SLA, authority, V1, Kafka, Redis, SQLite, Trading System, alpha or +order path changes are in scope. + +**C2 no-session TRADE race source gate (`PASS / ONE IMMUTABLE CLIENT +RECEIPT NEXT`, 2026-09-03).** The acceptance-only retry now honors the +declared consumer contract. It can make one ordinary strict snapshot re-read +for a TRADE with no session SLA only after the typed status proves fresh, +executable, complete, gap-free and `NOT_APPLICABLE` session semantics. It +still immediately rejects a quiet/stale event or a disconnected state, and +the re-read remains subject to normal strict freshness. Focused regressions +cover the allowed race plus both rejected states; the complete C2 +domain/receipt/handoff suite ran from the existing immutable read-only, +network-disabled image: **79 passed in 15.028s**. + +This is client-harness source only. Exactly one new immutable Python image is +now required because the disposable C2 client executes this changed source; +no reader, stream, Rust, ingestor, projector, V1 or other runtime role needs +an image or config change. The next permitted operation is one fresh C2 +four-identity/299-product/300-second no-order receipt from that image. Its +previous trust/admission roll remains the serving runtime and V1 remains the +explicit fallback/rollback route. + +**C2 immutable client receipt (`IN PROGRESS / ONE CLIENT IMAGE`, 2026-09-03).** +The approved closure action is narrowed to one immutable Python client image +from source revision `f2e5654`, tagged `qdl-v2-python:2.0.13-f2e5654`, then +one fresh disposable four-identity/299-product/true-300-second C2 receipt. +The image is not a serving-role rollout: query/stream retain their current +`qdl-v2-python:2.0.12-8ba4165` image and all V2 data-plane roles, V1, Kafka, +Redis, SQLite, Trading System, alpha and order paths remain unchanged. The +client mounts only the predeclared read-only evidence/runtime/ram-backed +credential staging inputs, has no provider or order authority, and self-removes +on completion. A nonzero receipt remains terminal fail-closed; rollback is +removing that disposable client and retaining the current V1 fallback route. + +**C2 native-BASIS admission correction (`IN PROGRESS / HARNESS-ONLY`, +2026-09-03).** The first immutable `2.0.13-f2e5654` receipt passed mTLS and +the repaired TRADE-session path, then stopped fail-closed while reading the +first `BINANCE.USDM.PERPETUAL.BTC-USDT / BASIS` reference product. The typed +public response was `SOURCE_UNAVAILABLE` with `reference batch provider lane +did not complete`; no receipt, order, signal, sizing, Gateway/Risk or runtime +mutation occurred, and the RAM-only credential staging tree was removed. + +The exact cause is C2-client concurrency, not a provider defect: each +consumer/replica independently starts its singleton Binance native-BASIS +batch, while the declared shared Rust `REFERENCE_NATIVE_BASIS` lane correctly +allows one in-flight request. The existing per-batch isolation and one typed +cooldown retry are therefore insufficient when both replicas and multiple +identities race the same lane. The narrow correction is to share one C2-local +semaphore across *only* these singleton native-BASIS batches, for opening and +closing reads. It preserves the Rust admission authority, all real-provider +requests, normal reference parallelism, strict errors and the one bounded +cooldown retry. Tests must prove native requests serialize across replicas and +ordinary reference batches remain outside that lane. No provider policy, +runtime role/image/config, manifest, freshness SLA, V1, Kafka, Redis, SQLite, +Trading System, alpha or order-path change is in scope. + +**C2 native-BASIS serialization source gate (`PASS / ONE REPLACEMENT CLIENT +IMAGE NEXT`, 2026-09-03).** The C2 harness now shares one local semaphore for +the Rust-admitted singleton Binance native-BASIS batch across all four +identities and both V2 reader replicas, in both opening and closing reads. +Ordinary reference batches remain under their existing bounded concurrency and +do not wait on that native lane. The shared classifier moved into the +Reference/L2 acceptance contract so batch construction and C2 use the exact +same definition. Source-only regression ran from immutable +`qdl-v2-python:2.0.13-f2e5654`, source mounted read-only, network disabled, +UID/GID `10001`, capability-free and tmpfs-only scratch: **99 passed, +25.533s**. The matrix includes the new concurrent native-lane and +ordinary-reference-not-blocked regressions plus C2 scope/receipt, V1 fallback, +Reference/L2, rollout and Rust-admission binding tests. A broad `compileall` +attempt was intentionally rejected by the read-only source mount because it +writes `.pyc`; it made no source/runtime mutation and is not a code failure. + +This source slice needs one replacement immutable disposable-client image +after commit. It does not roll a reader, stream, Rust core, ingestor, +projector, V1 or any deployed service. The only next runtime action remains +one fresh C2 receipt with the replacement client; V1 remains rollback. + +**C2 replacement-client runtime trace (`FAIL-CLOSED / REAL DATA-PLANE +DIAGNOSIS`, 2026-09-03).** Immutable disposable client +`qdl-v2-python:2.0.14-1c135af` (digest +`sha256:ac6125fc496f9ea0d59a164903906c5eb560141a42086c1013617c9f29be945f`) +passed the repaired mTLS and serialized native-BASIS entry path, then stopped +before its 300-second observation because the first governed +`BINANCE.USDM.PERPETUAL.BTC-USDT / TRADE` status was truthfully +`STALE/LAST_EVENT_STALE`. Both reader replicas reported the same stale typed +state; no synthetic event, SLA relaxation, order, signal, sizing, Gateway/Risk +or durable-state mutation was used to force progress. The disposable client +self-removed and its RAM-only credential stage was removed. + +Read-only runtime trace then proved the BTC trade binding is present in the +active Binance ingestor, all three ingestor lanes and all three projectors are +running with restart `0` and no OOM, and provider-session liveness later resumed +on every Binance lane. This rules out a missing demand binding or a permanently +dead process, but does **not** certify continuity: the failed C2 receipt remains +terminal until the raw-to-canonical-to-cache path is traced and a single fresh +receipt reaches its full 300-second observation. The next scope is strictly +read-only offsets/cache/lineage diagnosis followed by the smallest proven +shared recovery or source correction; V1, Kafka topology/offsets, Redis, +SQLite, Trading System, alpha and order paths remain excluded unless an exact +approved recovery is required. + +**Kafka diagnostic recovery (`APPROVED / NARROW RUNTIME REPAIR`, +2026-09-03).** The Java Kafka administrative CLI used for a read-only offset +inspection was invoked inside the memory-bounded `kafka1` broker container and +exceeded its `768MiB` container limit, leaving only `kafka1` in +`exited/OOMKilled` state. This is an operational error in the diagnostic +method, not a provider, data-contract or consumer defect. Recover only that +existing stopped container with `docker start`, retaining its current image, +network, Kafka data volume, TLS and KRaft identity exactly as-is. No recreate, +topic/offset/ACL mutation, Redis/SQLite operation, V1, Rust, ingestor, +projector, Trading System, alpha or order-path action is permitted. Verify all +three broker health checks and the existing core/projector roles afterward; +rollback is stopping only the restarted broker if it cannot rejoin. Future +offset diagnostics use the existing bounded librdkafka client outside broker +JVM memory, never a Java CLI inside a broker container. + +**Shared Rust-core transient receive recovery (`IN PROGRESS / SOURCE-ONLY`, +2026-09-03).** The failed C2 trace and bounded structured-log evidence show +retryable librdkafka receive failures (including a transient resolver failure) +reach `bridge.next()?`, which currently tears down the entire transactional +generation before the outer retry loop runs. That behavior is correct for a +transaction/commit/fencing failure, but it unnecessarily forces a cooperative +group rebalance for a recoverable receive-side hiccup and can create a +freshness gap in otherwise healthy TRADE bindings. + +The narrow correction is confined to `qdl-realtime-core`: retry a bounded +retryable *receive* error in the current bridge/generation with the existing +backoff policy and shutdown awareness. Persistent receive errors still return +to the existing outer generation retry; all commit, output, authority, +quarantine and non-retryable errors retain their present fail-closed behavior. +No provider adapter, subscription, manifest, Kafka topology/offset, Redis, +SQLite, V1, query/stream, Trading System, alpha or order path changes are in +scope. Source exit requires focused policy regressions plus the existing Rust +core suite. A later, separately journaled bounded roll of only the three Rust +core roles is allowed only if source tests pass; it must retain the current +runtime JSON, image rollback coordinate and all durable state. A fresh single +C2 receipt remains the only acceptance gate after that recovery. + +Implementation is now present in the isolated `dev` worktree: retryable +`bridge.next()` failures retain the current generation for at most three +backoff attempts, emit bounded structured receive diagnostics, and remain +shutdown-aware; an error while filling a batch defers that batch rather than +tearing down the bridge. The explicit policy regression covers retryable, +bounded-exhaustion and non-retryable cases. `git diff --check` passes. The +host has no Cargo toolchain, and the otherwise isolated, network-disabled +existing-builder test invocation was blocked before execution by the external +command-approval service returning HTTP `404`; therefore no Rust test or +runtime roll is claimed here. No runtime state was changed by this source +slice. The next action is to rerun exactly that existing source-only Rust +formatter/clippy/test suite when the approved Docker executor is available; +only then may the correction be committed and the separately bounded core roll +and one C2 receipt proceed. + +The sandboxed source executor also cannot reach the host-published V2 ports +`18201`, `18202`, `18210` or `18211` (all returned local connect failure from +its isolated network namespace). That is not health evidence for or against +the host runtime and must not be interpreted as a V2 outage. Host-runtime +verification remains limited to the approved Docker executor or an operator +shell in the host namespace; no fallback probe may substitute a synthetic or +different-network result for the C2 receipt. + +**Shared Rust-core transient receive source exit (`PASS / COMMIT AND NARROW +ROLL NEXT`, 2026-09-04).** The correction was compiled in the pinned Rust +`1.82` Docker builder only; the builder is a disposable test artifact and no +running service, Kafka, Redis, SQLite, V1, Trading System, alpha or order path +was changed. Exact evidence: + +- `docker build --target builder -f Dockerfile.phase8-rust .` completed the + locked release build including `qdl-realtime-core` successfully. +- `rustfmt --edition 2021 --check rust/qdl-kafka/src/bin/qdl-realtime-core.rs` + passed from a read-only, network-disabled container. +- `cargo clippy -p qdl-kafka --bin qdl-realtime-core --locked --offline -- + -D warnings` passed in `50.38s` in the same isolated builder. +- `cargo test -p qdl-kafka --bin qdl-realtime-core --locked --offline` passed + `5/5`, including retryable receive, bounded exhaustion and non-retryable + policy. `git diff --check` also passes. + +The broad workspace `cargo fmt --all -- --check` remains non-green solely +because pre-existing formatting deviations in +`qdl-native-raw-ingestor.rs` are outside this correction; that file was not +changed. The changed realtime-core file is format-clean and the focused +compile/lint/test gate is sufficient for this narrow slice. Commit only the +two tracked source/journal files with the configured BobbyAxerol identity. +After commit, build one immutable Rust runtime image from that commit, retain +the current `qdl-v2-rust:2.0.12-8ba4165@sha256:d86f0e832ba9...` image as +rollback, and rolling-recreate exactly the existing three core containers. +They retain current runtime JSON/TLS/state mounts and all durable state. A +single fresh disposable C2 no-order receipt of exactly `300s` is the only +post-roll acceptance; any failed receipt remains terminal and rolls those +three cores back without touching V1 or consumers. + +**Shared Rust-core transient receive rolling packet (`APPROVED / APPLIED`, +2026-09-04).** Source commit `5875d35aff3710be884eefa7bce9f8463aa73bcd` +was built as immutable +`qdl-v2-rust:2.0.14-5875d35@sha256:6d0668f1ded3648eb727d07872175b528ee38e4a01c1ae041e9469f354de168a`. +The exact rollback image is the three cores' pre-roll +`sha256:d86f0e832ba945d302fd3f782e26fd41c5b08709a80f6de16bdd36af5ed86983`. +Only these existing services were rolling-recreated, one healthy replica at a +time: `rust_core`, `rust_core_2`, `rust_core_3`. Their existing private runtime +JSON mounts remain byte-identical: `core.json` +`4ff72e09649a7d00d02b8b006e9dfc1bd82ff6b5a6718a02a03fcc07c629d74f`, +`core-002.json` +`fe85911773eedda38a9fc84ef485c1e2bd1a5bd115a1268bac361075972a25ce`, and +`core-003.json` +`84081f68a6290bd130d770e1d36b197aa52d8df71c8a249bea3c0baa4a6dd946`. +The existing TLS volume, state mounts, Kafka group/topic/offsets, Redis, +SQLite, V1, ingestors, projectors, query/stream roles, Trading System, alpha +and order paths are excluded. All three replacement cores start +`RUST_PRIMARY`, `bindings=182`, `restart=0`, `OOMKilled=false`; the first +retains the already-enabled private provider-admission listener. Rollback is a +rolling recreate of these same three names with the exact old image and the +same current runtime config; no durable reset or topology mutation is part of +either direction. + +The sole remaining receipt is one new disposable C2 client namespace using the +immutable Python client `qdl-v2-python:2.0.14-1c135af`, exactly four paper +identities (monitoring, Trading System paper, Binance alpha paper, OKX alpha +paper), all governed `299` products, V2 query/stream, and only its declared +local V1 fallback-return drill. It observes exactly `300s`, has no provider +credentials, Docker socket, Kafka, Redis, SQLite, Gateway/Risk, signal, sizing +or order authority, and self-removes. Its credential staging is RAM-only and +only bounded payload-free receipt/error hashes persist. A nonzero exit is +terminal fail-closed and triggers only the stated three-core image rollback; +it never causes a synthetic event, policy/SLA relaxation, consumer mutation or +retry ceremony. + +**C2 client bootstrap preflight (`NO RECEIPT / HARNESS ENTRYPOINT ONLY`, +2026-09-04).** The first launch attempt did not create a client process: OCI +rejected direct execution of the read-only bind-mounted bootstrap script with +`permission denied` before the process, network namespace, mTLS/JWT material, +V2/V1 request, provider, data-plane or order action existed. The disposable +container name was not retained and this is not a C2 retry or acceptance +result. The sole harness correction is to invoke the same immutable, +read-only bootstrap bytes through `/bin/sh /bootstrap-c2.sh`; it preserves the +same mount set, capabilities, tmpfs staging, identities, 300-second scope and +self-removal. No runtime role, image, config, credential, consumer or durable +state changes. + +The shell-interpreter preflight then reached the bootstrap but still stopped +before Python, mTLS/JWT construction or any V2/V1 request: the inherited +mode-`0700` `run-c2.sh` could not be read by the intentionally dropped UID +`10001`, and the capability-restricted bootstrap parent could not synthesize +an exit code into that UID's tmpfs directory. This is still no receipt and no +data-plane action. The new disposable namespace corrects only those two +harness mechanics: its non-secret run script is mode `0755`, and bootstrap +writes a fallback exit code through the same already-dropped UID `10001`. +The actual receipt starts in a new evidence namespace, preserving a clear +terminal preflight trail rather than overwriting it. + +**C2 consumer-network correction (`FAIL-CLOSED / HARNESS ATTACHMENT ONLY`, +2026-09-04).** The first client that reached Python used +`stable_internal` and stopped at the first gRPC stream open because +`qdl-v2-stream-b` is deliberately published only on the declared external +consumer network. The typed error is DNS `UNAVAILABLE`, not a stream process, +Rust core, TLS, data freshness or provider failure. Read-only inspection proves +the sealed `QDL_STABLE_CONSUMER_NETWORK=executor_network` and that exact +network contains `data_layer_service`, both V2 query replicas, and both stream +replicas with their public aliases. The failed client self-removed; its security +evidence confirms UID `10001`, no effective/inheritable/ambient capability and +`NoNewPrivs=1`; no order, signal, sizing, Gateway/Risk or durable mutation +occurred. The fresh receipt changes only its Docker network attachment to the +declared `executor_network`; no service or source code is recreated or changed. + +**C2 bounded product-concurrency correction (`FAIL-CLOSED / HARNESS SCHEDULING +ONLY`, 2026-09-04).** The correctly attached fresh C2 client reached its +immutable Python acceptance code and then exhausted the fixed `900s` opening +deadline while tasks were queued on its own `product_semaphore`. Its sealed +run artifact requested concurrency `2` for all governed `299` products; the +failure occurred before any typed per-product freshness, gap, session, +provider, TLS, cursor or fallback result was produced. The client self-removed +with exit `1`; its UID/capability evidence remains `10001`/no capabilities/ +`NoNewPrivs=1`, and no order, signal, sizing, Gateway/Risk or durable mutation +occurred. This is not a Rust-core or data-plane failure and is not a reason to +relax a data SLA. + +The one permitted fresh receipt changes only the disposable C2 artifact from +`--concurrency 2` to `--concurrency 4`: that equals the already sealed +reference-batch maximum, retains the existing 75% per-consumer request quota +and native-basis semaphore, and neither raises provider concurrency nor changes +any runtime service. It uses the same four paper identities, `299` governed +products, `executor_network`, V2 routes, declared V1 fallback-return drill and +exact `300s` observation. Any nonzero result remains terminal and triggers the +recorded three-core rollback only; a passing receipt is the sole remaining +closure gate for this narrowly approved repair. + +**C2 bootstrap transport preflight (`PASS / NO NETWORK`, 2026-09-04).** The +same immutable client image, read-only root, root-only bootstrap, RAM-only +credential staging, UID-`10001` capability drop, evidence mounts and resource +limits completed with `--network none`; it produced only `exit_code=0` and the +expected capability record. It opened no V1/V2/provider endpoint and changed +no runtime state. A first attached executor-network launcher left no receipt, +stderr or exit artifact before Docker auto-removal, so it is not counted as a +C2 result and has no data-plane evidence. The one fresh C2 receipt is launched +detached solely to decouple the 300-second observation from the shell transport; +the same immutable image, identity/mount set, `executor_network`, quota and +no-order constraints remain in force. No service, source or durable state +changes. + +**C2 real acceptance after receive recovery (`FAIL-CLOSED / ROLLBACK REQUIRED`, +2026-09-04).** The detached, auto-removed C2 client completed the reviewed +root-to-UID-`10001` boundary (`CapEff=0`, `CapAmb=0`, `NoNewPrivs=1`) and made +real V2 reads through the declared `executor_network`; it had no provider, +Docker, Kafka, Redis, SQLite, Gateway/Risk, signal, sizing or order authority. +It stopped before the 300-second observation during initial reference +validation for demand UID `8aedd349-6999-5874-b0dd-34c6451c0b3a`, product +`MARK_INDEX_PRICE`: the returned identity is exact, but its typed availability +is `DATA_STALE` with `reference provider result exceeds the declared freshness +bound`. This is a real V2 quality failure, not a timeout, retry, identity +cross-mix, TLS, cursor, Rust receive-recovery or resource/OOM result. The +receipt is terminal; its client self-removed and left only bounded security and +error evidence. + +All three rolled Rust cores remain `running`, `restart=0`, `OOMKilled=false` +on `sha256:6d0668f...de168a`; no core error was observed. Nevertheless the +approved packet makes a nonzero C2 terminal: roll back only `rust_core`, +`rust_core_2` and `rust_core_3` to +`sha256:d86f0e832ba945d302fd3f782e26fd41c5b08709a80f6de16bdd36af5ed86983`, +retaining byte-identical runtime JSON/TLS/state mounts. Do not change V1, +Kafka topology/offsets, Redis, SQLite, ingestors, projectors, query/stream, +Trading System, alpha or order path. This closes the receive-recovery runtime +packet as `SOURCE PASS / RUNTIME C2 NOT CERTIFIED`; the MARK_INDEX freshness +defect is a separate data-quality scope and blocks closure honestly. + +**Receive-recovery rollback exit and scoped cleanup (`APPLIED / CLOSURE +BLOCKED`, 2026-09-04).** The three rollback recreates completed serially and +all now run the retained prior image +`sha256:d86f0e832ba945d302fd3f782e26fd41c5b08709a80f6de16bdd36af5ed86983` +with `restart=0` and `OOMKilled=false`. No C2 container remains. Before +cleanup, Docker reported `14GB` images (`4.974GB` reclaimable) and `8.365GB` +BuildKit cache (`5.336GB` reclaimable). The only disposable artifacts from this +slice are the no-longer-referenced immutable candidate +`qdl-v2-rust:2.0.14-5875d35@sha256:6d0668f...de168a` and builder +`qdl-v2-rust-builder:receive-recovery-1c135af`; retain the active rollback +image, all running V1/V2 images, volumes, networks, runtime state and compact +receipt evidence. No broad prune, cache purge, volume/network/data deletion or +worktree removal is in scope. Post-cleanup disk and runtime health are recorded +before the journal commit. + +**Scoped cleanup result (`PASS / NO BROAD PRUNE`, 2026-09-04).** Docker confirmed +neither disposable image had a container reference, then removed exactly +`qdl-v2-rust:2.0.14-5875d35` and +`qdl-v2-rust-builder:receive-recovery-1c135af`. Image usage fell from `14GB` +to `10.84GB` (reclaimable image space from `4.974GB` to `1.808GB`). The active +rollback image remains present; every V1/V2 running image, volume, network, +runtime directory and compact evidence namespace is retained. BuildKit cache +is intentionally untouched (`8.365GB`, `5.421GB` reported reclaimable): it is +shared across current projects and a broad cache prune is outside this narrow +approval. Final post-cleanup checks show all three rollback cores running on +`sha256:d86f0e...86983`, `restart=0`, `OOMKilled=false`. + +**MARK_INDEX_PRICE exact-scope diagnosis (`IN PROGRESS / READ-ONLY`, +2026-09-04).** The terminal C2 product is Binance USD-M `DOGEUSDT`, demand UID +`8aedd349-6999-5874-b0dd-34c6451c0b3a`, `MARK_INDEX_PRICE`; this is a bounded +on-demand provider snapshot behind `/v2/market-data/reference:batch`, not a +Kafka BAR/L2 materialization defect. The approved diagnostic reads the exact +signed V2 requirement through both existing query replicas and records only +typed status, provider-observed age, response/receive age, cache provenance, +timestamp origin and lineage. It does not relax the 60-second contract, make a +synthetic observation, use V1, recreate any role, or mutate Kafka, Redis, +SQLite, Trading System, alpha, signal, sizing or order state. + +Exit decision: if both replicas return current V2 data, repair the C2/reference +batch lifecycle so a fresh execution-grade snapshot cannot age behind unrelated +history work; if either replica returns stale V2 data, repair only the shared +query/reference timestamp or cache path and add focused fresh/missing/stale and +two-replica regression coverage. The Rust receive-recovery patch is independent +of this defect and is not re-rolled merely to retry the same reference failure. +Only after the exact fault is fixed and source gates pass may one immutable, +bounded rollout packet and one fresh 300-second C2 receipt be prepared. + +**MARK_INDEX_PRICE exact-scope diagnostic result (`PASS / SOURCE REPAIR +REQUIRED`, 2026-09-04).** A disposable, signed V2-only read used the declared +`alpha.binance.paper.stable` identity and the exact DOGE requirement through +both live query replicas. `query_v2_1` returned `OK` in `172ms` with +provider-observed age `407ms`; `query_v2_2` returned `OK` in `85ms` with +provider-observed age `494ms`. Both retained the exact demand UID, native +symbol `DOGEUSDT`, `timestamp_origin=PROVIDER`, cache miss provenance and only +`/fapi/v1/premiumIndex` lineage. This rules out a missing catalog identity, +permanent Binance provider staleness, replica cross-mix, V1 fallback and +Kafka/L2 materialization as the C2 root cause. + +The remaining defect is bounded reference-snapshot recovery: a transient stale +provider MARK/INDEX row can terminalize the whole C2 batch even though the +same provider lane is current immediately afterward. The approved source scope +is therefore one cache-bypassing re-read only for a non-history +`MARK_INDEX_PRICE` requirement with a declared freshness bound. It does not +relax freshness, substitute a source, synthesize data, or retry any other +reference product. The re-read must independently satisfy the original bound; +otherwise the typed `DATA_STALE` result remains fail-closed. Focused tests must +cover stale-then-current pass, stale-then-stale fail, and the existing +batch-aging recovery without changing history/reference semantics. No runtime +role, V1, Kafka, Redis, SQLite, Trading System, alpha or order state is changed +by this source slice. + +**MARK_INDEX_PRICE bounded recovery source exit (`PASS / RUNTIME C2 PENDING`, +2026-09-04).** `qdl/query/service.py` now re-reads only a non-history +`MARK_INDEX_PRICE` result that fails its declared freshness bound, using the +existing bounded executor, same provider lane and `bypass_cache=True`. The +re-read is exactly once: a current second result is returned normally; a second +stale result remains the same typed `DATA_STALE`. Existing current-at-receipt +batch-aging recovery remains unchanged. Funding, OI, long/short, taker, basis, +metadata, history and V1 never enter this new branch. + +Source-only gates passed in an isolated, read-only, network-disabled container +using the active immutable Python runtime image as the dependency carrier: +`py_compile qdl/query/service.py tests/test_phase113_reference_v2.py`, then +`62/62` tests across `test_phase104_reference_batch`, +`test_phase113_reference_v2`, `test_reference_l2_consumer_acceptance` and +`test_phase115c_five_liquid_handoff`. The matrix covers decimal/unit fidelity, +identity isolation, provider lane bounds, cached-snapshot aging, transient +stale-to-current MARK/INDEX recovery, stale-to-stale fail-closed behavior, +non-MARK stale no-retry, API/SDK serialization, L2 generation/gap guards and +five-symbol manifest admission. No provider call, V1 fallback, Kafka, Redis, +SQLite, runtime role, Trading System, alpha, signal, sizing or order state was +changed. Disposable test containers self-removed; no image was built and no +cache/image cleanup is required for this source slice. Post-test Docker +inventory is unchanged at `10.84GB` images (`1.808GB` reclaimable), `8.365GB` +BuildKit cache (`5.421GB` reclaimable), `47` volumes (`13` active) and no +reference test container. The active V1/V2 images, retained Rust rollback +image, runtime state, TLS, volumes and networks remain the explicit retention +set; shared BuildKit cache is outside this source-only cleanup boundary. + +**Next approved-boundary packet (`PENDING OWNER RELEASE APPROVAL`).** Build one +immutable Python image from the committed source revision, retain current +`sha256:bd0163fd76b0...` as rollback, then rolling-recreate only +`query_v2_1` and `query_v2_2` with their existing runtime/TLS/state mounts. +After both are healthy, run one exact 300-second C2 no-order receipt with the +existing consumer identities and the same V2 manifest; verify DOGE +`MARK_INDEX_PRICE` through both replicas, final BAR/quote/trade/reference, +signed cursor/reconnect and V1 fallback-return policy. Any nonzero result rolls +only those two query roles back to `sha256:bd0163fd76b0...`; V1, Rust cores, +Kafka offsets/topology, Redis, SQLite, ingestors, projectors, stream roles, +Trading System, alpha and all order paths remain untouched. This packet is the +remaining certification gate before release; no SLA relaxation or retry loop is +permitted. + +**MARK_INDEX_PRICE bounded recovery runtime packet (`APPROVED / APPLIED`, +2026-09-04).** Owner approved exactly the preceding packet: build one immutable +Python image from source commit `f5b14f141a5a36abee7ddda543b274f3feccb4bf`, +rolling-recreate only `query_v2_1` then `query_v2_2` with their existing +runtime/TLS/state mounts, and run one C2 no-order receipt with a true +300-second observation. The only authorized rollback is those two query roles +to `sha256:bd0163fd76b0...`. No V1 role, Rust core, Kafka topology/offset, +Redis, SQLite, ingestor, projector, stream role, Trading System, alpha or +order/signal/sizing path is in this blast radius. A passing receipt permits +release certification; a failing receipt triggers the declared two-role +rollback and remains fail-closed. After either terminal result, remove only +the disposable C2 client and test-only build artifacts not retained as the +named active/rollback image set, then record pre/post disk evidence. + +**MARK_INDEX_PRICE bounded recovery rollout and C2 exit (`SOURCE PASS / +RUNTIME NOT CERTIFIED`, 2026-09-04).** Immutable Python image +`qdl-v2-python:2.0.13-f5b14f1@sha256:55a63fb82381829c7089075c64a1276f5fa865f8a7031a5930acc2054aa23a42` +was built from exactly `f5b14f141a5a36abee7ddda543b274f3feccb4bf`; OCI +revision/version and non-root `qdl:qdl` user were verified. Its no-source-mount, +read-only, network-disabled `py_compile` plus four-module regression matrix +passed `62/62` in `5.849s`. + +Only `query_v2_1` then `query_v2_2` were rolling-recreated to that image after +Compose preflight; each reached `healthy`, `restart=0`. Two disposable launcher +preflights stopped before an endpoint request: the first copied a non-secret +runner at mode `0700`, and the second lacked the read-only `/runtime` authority +mount. A fresh namespace corrected only those launch mechanics (`0755` public +runner and the existing `/runtime` read-only mount); it dropped to UID `10001` +with no effective/inheritable/ambient capability and `NoNewPrivs=1`. + +That third receipt reached real V2 reads but stopped fail-closed before the +300-second observation and before an acceptance JSON could be emitted. The +exact typed cause is `RESOURCE_EXHAUSTED: consumer request quota is exhausted` +while the C2 harness opens `alpha.binance.paper.stable / BINANCE.USDM.PERPETUAL +DOGE-USDT / BOOK_SNAPSHOT`. This is not a `MARK_INDEX_PRICE`, DOGE catalog, +provider freshness, V1 fallback, Rust, Kafka or data-materialization defect. +It exposes a harness correctness gap: the C2 pacer bounds query REST requests +but does not pace its own concurrent gRPC stream opens against the same +per-identity quota. No SLA was relaxed and no synthetic/provider-direct data +was used. The disposable client has no Docker socket, Gateway/Risk, broker, +signal, sizing or order authority; only its approved bounded V2 query/stream +and normal quota/audit activity occurred. + +Per the approved nonzero exit rule, only `query_v2_1` then `query_v2_2` were +rolled back to `sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`. +Both are now `running`, `healthy`, `restart=0`. V1, Rust cores, Kafka +topology/offsets, Redis data, SQLite, ingestors, projectors, stream roles, +Trading System, alpha and order paths were not recreated or changed. All C2 +containers self-removed. The unattached candidate image was removed exactly; +Docker images returned from `11.5GB` to the pre-packet `10.84GB`. +BuildKit cache is `9.049GB` (`6.106GB` reclaimable), shared and intentionally +not broadly pruned without a separate approval. The compact three receipt +namespaces are retained as failure evidence; they contain no copied secret +material. The narrow next source scope, if approved, is to apply the existing +per-identity C2 quota pacer to stream-open scheduling, add quota/stream +regressions, then build one replacement image and run exactly one fresh C2. +Release remains blocked until that single receipt reaches `300s` and passes. + +**C2 stream-open quota pacing source gate (`PASS / IMMUTABLE IMAGE PENDING`, +2026-09-04).** `_PacedStreamTransport` now wraps only the disposable C2 SDK +client's `stream_transport`. It reserves the existing shared identity pacer at +async-iterator start, so an initial subscription and every SDK reconnect share +the same allowance as C2 REST warmup/snapshot/reference reads. The wrapper +passes frames and failures through unchanged; it cannot turn a rejected or +stale stream into data. `_paced_client_factory` applies the two wrappers to the +same client instance, while distinct C2 consumer identities still receive +separate pacers. + +The standard isolated source matrix used active immutable image `bd0163fd76b0`, +with a read-only source mount, `--network none`, UID/GID `10001`, no Linux +capabilities, no-new-privileges and tmpfs-only bytecode. It ran +`py_compile scripts/phase105_consumer_v2_identity_acceptance.py +tests/test_phase105_identity_acceptance.py`, then `127/127` unittest cases in +`test_phase104_reference_batch`, `test_phase113_reference_v2`, +`test_reference_l2_consumer_acceptance`, `test_phase115c_five_liquid_handoff`, +`test_phase105_identity_acceptance` and `test_phase103_consumer_receipt_harness` +in `16.053s`. New deterministic cases prove factory wiring, shared REST/stream +spacing, independent identity pacers and fail-closed stream errors. No network, +provider, V1 fallback, Kafka, Redis, SQLite, service, Trading System, alpha, +signal, sizing or order mutation occurred; the disposable test client removed +itself. `git diff --check` passes. Next and only next action is to commit this +source slice, build one immutable image from its SHA, repeat this exact matrix +without a source mount, then execute the recorded two-query-role C2 packet. + +**C2 stream-open quota pacing immutable-image gate (`PASS / QUERY ROLLOUT +PENDING`, 2026-09-04).** Immutable candidate +`qdl-v2-python:2.0.14-a8fdb55@sha256:be78f3b864b44809354a26fa3d9f2e28b49c15f03ff325188cccbc1d1f0eb341` +was built from source commit `a8fdb5556207888958b97c8383b4baa6fde9752b`. +OCI revision/version labels match that source and `qdl:qdl` remains the runtime +user. The same `py_compile` plus six-module C2 matrix ran from the immutable +image with no source mount, `--network none`, read-only root, UID/GID `10001`, +no capabilities/no-new-privileges and tmpfs-only bytecode: `127/127 PASS` in +`15.714s`. No provider, V1, Kafka, Redis, SQLite, runtime service, Trading +System, alpha, signal, sizing or order path was touched; the test client +self-removed. + +The exact remaining approved action is serial rolling recreate of only +`query_v2_1` and `query_v2_2` to this digest with unchanged runtime/TLS/state +mounts, readiness verification after each, then one fresh disposable C2 +four-identity 300-second no-order receipt. A nonzero receipt triggers only the +named two-query rollback digest in the preceding packet; no second C2 retry or +scope expansion follows automatically. + +**C2 bootstrap capability preflight (`PASS / NO RECEIPT`, 2026-09-04).** The +first detached client self-removed before writing any security, exit or C2 +evidence. It therefore did not reach Python, a V1/V2 endpoint, a provider, +cursor, fallback drill or the 300-second observation and is not a C2 result. +The launcher was missing the two temporary bootstrap capabilities needed solely +to drop from root to UID/GID `10001`. A disposable `--network none` preflight +with only `SETUID` and `SETGID` added proved the exact intended child boundary: +`uid=10001`, `CapEff=0`, `NoNewPrivs=1`. The next real C2 uses those two +bootstrap-only caps; its child still receives no effective/inheritable/ambient +capabilities. Nothing else in the packet changes, and the failed empty +namespace is retained as a launcher preflight record. + +**C2 read-only identity-mount preflight (`PASS / NO RECEIPT`, 2026-09-04).** The +second empty launcher namespace likewise stopped before Python/evidence because +the minimal root bootstrap could drop UID/GID but had no `DAC_OVERRIDE` to read +the declared mode-restricted identity files from its read-only `/v2state` mount. +This is a capability boundary of the disposable launcher, not an endpoint, +quota, data-plane or runtime failure. A second `--network none` preflight +proved that only `DAC_OVERRIDE`, `SETUID` and `SETGID` are sufficient to read +the exact declared alpha-Binance identity input, then drop to `uid=10001`, +`CapEff=0`, `NoNewPrivs=1`. The real C2 launcher is revised only to use those +three bootstrap-only capabilities; the child has no capability and the mount +remains read-only. It is still the first actual C2 receipt, not a retry of a +receipt, because neither prior launcher opened an endpoint or wrote C2 output. + +**C2 stream-open quota pacing repair (`APPROVED / IN PROGRESS`, 2026-09-04).** +Owner approved one final narrow closure repair: the existing acceptance-only +per-identity C2 pacer must govern gRPC stream opens as well as REST reads. The +source change is limited to the disposable C2 client factory: wrap its existing +`stream_transport.subscribe()` path so each initial subscription and every SDK +reconnect reserves the same manifest-derived identity budget already used by +`_PacedQueryTransport`. It does not alter a server quota, consumer manifest, +SDK/public endpoint semantics, provider concurrency, stream service, Rust, +Kafka, Redis, SQLite, V1 fallback policy, or runtime topology. Separate C2 +identities retain separate pacers; a transport exception must propagate +fail-closed unchanged. + +**Required source gates:** deterministic regressions prove same-identity stream +opens are serialized below the existing safe rate, stream and REST share one +budget, distinct identities do not share a local lock, and a failing stream +remains terminal. Run the existing focused C2/identity/reference/L2/five-symbol +matrix plus `py_compile` and `git diff --check` in the standard isolated, +network-disabled, read-only image environment. The detailed consumer-cutover +invariants remain those in architecture-guide Appendix J.7. + +**Approved runtime/rollback/cleanup packet:** build exactly one immutable Python +image from the tested source commit; rolling-recreate only `query_v2_1` then +`query_v2_2` with their existing runtime/TLS/state mounts; run exactly one +disposable, no-order, four-identity C2 receipt with a true `300s` observation. +If it exits nonzero, roll back only those two query roles to +`sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`. +V1, Rust, Kafka topology/offsets, Redis, SQLite, ingestors, projectors, stream +roles, Trading System, alpha and all order/signal/sizing paths remain outside +the blast radius. Retain only the active candidate plus named rollback image; +remove the self-removed C2 client and any unattached candidate after a failed +receipt, record pre/post Docker disk evidence, and do not broad-prune shared +BuildKit cache. + +**C2 stream-pacing receipt and bounded MARK/INDEX assembly correction (`IN +PROGRESS / SAME CLOSURE`, 2026-09-04).** The first real receipt using the +stream-paced candidate reached authenticated V2 reads and the disposable child +correctly ran as UID `10001` with no effective/inheritable/ambient capability +and `NoNewPrivs=1`. It failed before the 300-second observation on Binance +USD-M `DOGEUSDT` `MARK_INDEX_PRICE`: the typed result was `DATA_STALE` because +the final batch assembly evaluated a refreshed mark snapshot after other +refresh work had completed. Direct bounded read-only Binance observations +showed provider timestamps 0--18ms old, so this is neither provider staleness +nor a reason to relax the declared 2-second SLA. + +The approved closure repair remains narrow and does not create a new phase: +for an already-admitted, non-history `MARK_INDEX_PRICE` item that needs the +existing one cache-bypass recovery, execute and validate that exact item at its +own response-assembly turn. It remains on the existing bounded executor and +same venue/provider lane, is still allowed exactly one recovery read, and a +second stale/error result remains fail-closed. This prevents a fresh DOGE mark +from aging behind unrelated MARK refresh candidates without changing cache TTL, +provider admission, public API/SDK contracts, quota policy, V1 fallback, +Rust, Kafka, Redis, SQLite, manifests, topology or any order path. + +**Required correction tests and exit:** add a deterministic multi-symbol MARK +batch regression which advances the test clock between refresh candidates and +proves every response is validated immediately after its own re-read; retain +the existing stale-to-current, stale-to-stale, non-MARK no-retry, shared +singleflight and identity/stream-quota cases. Run the focused six-module +isolated matrix and `py_compile` with network disabled before one replacement +immutable image. The only runtime action remains the already-approved serial +two-query-role rollout and one fresh C2 receipt. A nonzero receipt rolls back +only `query_v2_1` and `query_v2_2` to +`sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`. + +**C2 MARK/INDEX assembly source gate (`PASS / REPLACEMENT IMAGE PENDING`, +2026-09-04).** `V2QueryService.reference_data_batch_async()` no longer submits +all stale MARK/INDEX recoveries as one second batch. It first identifies only +the existing eligible one-read candidates, then at each candidate's response +assembly turn submits that exact item through the same bounded executor with +`bypass_cache=True` and validates it immediately. This keeps provider token +budgeting, venue isolation, request identity, result ordering and the exact +one-refresh limit intact. A second stale/error result is still returned as its +typed terminal problem; history and all non-MARK products retain the old path. + +The new deterministic two-symbol regression models the failure exactly: both +initial marks age after initial work, while a legacy two-item refresh would age +again before response assembly. Per-item refresh/validation returns two current +results in request order after four provider calls; the legacy grouped form +would fail `DATA_STALE`. Focused test first passed `16/16` in `0.319s`; the +complete isolated source matrix then passed `128/128` in `16.198s` after +`py_compile` of the query service and C2 source/tests. Both runs used the +existing immutable dependency image, source mounted read-only, `--network +none`, read-only root, UID/GID `10001`, no Linux capabilities, +no-new-privileges and tmpfs-only bytecode/cache state. The launch explicitly +preserved `/opt/venv/bin` rather than invoking login-shell PATH reset. No +provider, V1, Kafka, Redis, SQLite, service, Trading System, alpha, signal, +sizing or order mutation occurred; disposable test containers removed +themselves. `git diff --check` passes. + +**Next exact action:** commit this tested source-and-journal slice, build one +replacement immutable Python image from that commit, repeat the same 128-case +matrix with no source mount, then use the already-approved serial two-query +rollout and exactly one fresh C2 300-second receipt. The prior unattached +`2.0.14-a8fdb55` image remains test-only and will be removed after the +replacement source/image evidence is captured; no broad cache prune is in +scope. + +**C2 MARK/INDEX immutable image gate (`PASS / TWO-QUERY ROLLOUT PENDING`, +2026-09-04).** Tested source and journal were committed as +`5ba7342 fix(reference): validate mark refresh per item`. One replacement image +was built from exactly that revision: +`qdl-v2-python:2.0.15-5ba7342@sha256:f29ee76868fa38b4e540d5f906adb1db332ef298675a21f870229fb4474275e6`. +OCI revision/version labels are `5ba7342` / `2.0.15-5ba7342`, and the image +user remains `qdl:qdl`. With no source mount, the same read-only, +network-disabled, UID/GID-`10001`, no-capability, no-new-privileges, +tmpfs-only matrix passed `128/128` in `15.841s`. It included the new +multi-MARK assembly oracle plus C2 quota, reference, L2, five-symbol and +receipt-harness tests. No provider or runtime/data-plane was touched. + +Pre-rollout verification confirms both approved roles currently run the named +rollback digest `sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`, +are `healthy` with `restart=0`, and the sealed runtime env still names that +digest. The next command overrides only the image value for a serial +`query_v2_1`, then `query_v2_2`, recreate with their existing compose/runtime, +TLS and state mounts. It must verify health after each role before one +disposable 300-second C2 no-order receipt. A nonzero receipt restores only +those same roles to the verified rollback digest. + +**C2 MARK/INDEX rollout exit and quiet-final-BAR correction (`IN PROGRESS / +SAME CLOSURE`, 2026-09-04).** The two-query rollout to +`sha256:f29ee76868fa38b4e540d5f906adb1db332ef298675a21f870229fb4474275e6` +completed serially and both roles were `healthy`, `restart=0`. The one actual +four-identity C2 receipt then passed the earlier quota and DOGE MARK/INDEX +opening work, but failed before its 300-second observation on a different +typed harness transition: `alpha.okx.paper.stable`, `OKX.SWAP.PERPETUAL +ETH-USDT`, `BAR 5m` recorded `CURRENT_FINAL_BAR` for a valid quiet first +session and `EVENT_AFTER_REOPEN` when a real final BAR arrived after the signed +reopen. `_stream_handoff_mode()` incorrectly accepted the latter only after a +price/continuity initial session and rejected a current final BAR initial +session. This is an acceptance-harness state-transition defect, not provider +staleness, quota, MARK/INDEX, V1 fallback, Rust, catalog, materialization or +data loss. The terminal C2 client self-removed and wrote only scoped failure +evidence; no acceptance JSON was emitted. + +Per the approved nonzero rule, `query_v2_1` then `query_v2_2` were rolled back +serially to `sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`; +both are again `running`, `healthy`, `restart=0`. No other V2 role, V1, Rust, +Kafka, Redis, SQLite, provider, Trading System, alpha, signal, sizing or order +path changed. The only remaining source correction is to admit the valid +`CURRENT_FINAL_BAR -> EVENT_AFTER_REOPEN` no-cursor handoff for a non-execution +durable BAR, with a direct regression through `_stream_resume` and the existing +handoff classifier. It must not make an initial BAR executable, weaken signed +cursor control requirements, accept a gap/identity mismatch, or alter any +public/runtime contract. After focused source/image gates, a fresh C2 receipt +requires its own explicit owner packet because the declared one receipt has +already been consumed and failed closed. + +**Quiet-final-BAR handoff source gate (`PASS / REPLACEMENT IMAGE PENDING`, +2026-09-04).** `_stream_handoff_mode()` now recognizes the only missing valid +transition: a non-execution durable BAR first session proved by a strict +`CURRENT_FINAL_BAR` read, followed by a real `EVENT_AFTER_REOPEN` after signed +reopen. It returns the existing no-cursor live-event evidence label, while the +initial BAR remains non-executable and all cursor controls, identity, gap, +quality and finality checks remain unchanged. A new end-to-end `_stream_resume` +regression proves both signed session controls, the strict first final read, +the real reopened event and acknowledgement; the legacy classifier rejects +that exact evidence pair. The focused harness passed `47/47` in `11.893s`, and +the complete C2/reference/L2/five-symbol/identity matrix passed `129/129` in +`18.243s` after `py_compile`. Tests used the existing immutable dependency +image with source read-only, `--network none`, read-only root, UID/GID `10001`, +no capabilities/no-new-privileges and tmpfs-only bytecode/cache state. No +provider, V1, Kafka, Redis, SQLite, runtime role, Trading System, alpha, +signal, sizing or order mutation occurred; every test client self-removed. +`git diff --check` passes. + +**Decision boundary:** the previous C2 receipt is terminal and remains failure +evidence. This source correction is ready for a single replacement image, but +another 300-second C2 rollout is intentionally not started: it requires a new +explicit owner packet naming the replacement digest and the same two-query +rollback because the prior approved receipt was already consumed. No other +source or runtime scope is open. + +**Quiet-final-BAR immutable candidate and same-scope C2 packet (`APPROVED / +IN PROGRESS`, 2026-09-04).** The owner has standing approval to complete this +closure without a follow-up prompt, using the same bounded query-only blast +radius and rollback previously specified. Candidate +`qdl-v2-python:2.0.16-6d6cfca@sha256:077271ceb7cfb2aaf444e5a019237cab60d14e23882e7ff7513eed9fee0da39c` +was built from exactly commit `6d6cfca`; OCI labels match and user remains +`qdl:qdl`. Its no-source-mount, network-disabled, read-only, +UID/GID-`10001`, no-capability/no-new-privileges, tmpfs-only matrix passed +`129/129` in `18.011s`. + +The sole runtime packet serially recreates only `query_v2_1` then `query_v2_2` +with this digest and their unchanged runtime/TLS/state mounts, verifies each +role `healthy`/`restart=0`, and runs one disposable four-identity C2 no-order +receipt with a true 300-second observation. It uses the established read-only +identity bootstrap boundary and scoped evidence namespace. On nonzero, only +those two roles return serially to +`sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`. +No V1, Rust, Kafka topology/offsets, Redis, SQLite, ingestor, projector, +stream role, Trading System, alpha or order/signal/sizing path is authorized. + +**C2 strict-reference batch correction (`IN PROGRESS / SAME CLOSURE`, +2026-09-04).** The final candidate passed source/image gates and both query +replicas were healthy, but its one C2 receipt reached the reference plane and +failed before observation with `reference response exceeds its governed +freshness bound`. This is not the repaired DOGE query error: C2's +`trading-system.paper.stable` scope places ten independent execution +`MARK_INDEX_PRICE` snapshots, each with the declared `2,000ms` bound, in one +ordinary 12-item reference batch. Provider work and exact one-read recovery +remain bounded/fail-closed, but a batch receipt cannot honestly certify every +two-second snapshot at client receipt when earlier rows must wait behind later +strict rows. The failure is correct; it exposed an invalid certification batch +shape rather than grounds to relax the 2-second product contract. + +Per the same nonzero rule, only `query_v2_1` then `query_v2_2` were restored +serially to `sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`; +both are `healthy`, `restart=0`. The narrow source fix is acceptance-only: +`reference_acceptance_batches()` must isolate each `MARK_INDEX_PRICE` whose +declared freshness is at most `2,000ms`, just as it already isolates native +Binance basis. All looser reference products remain bounded 12-item batches; +no provider, cache, public batch contract, manifest, runtime role, quota, +SLA, V1/Rust/Kafka/Redis/SQLite, Trading System, alpha or order path changes. +Regression must prove complete identity preservation, native-basis isolation, +strict-Mark singleton isolation and ordinary batch bounds. It fixes request +shape only; every strict response still needs to meet its original 2-second +bound and a stale response remains terminal. + +**C2 strict-reference source gate (`PASS / IMMUTABLE CANDIDATE PENDING`, +2026-09-04).** `reference_acceptance_batches()` now isolates only +`MARK_INDEX_PRICE` products whose declared freshness is at most `2,000ms`, in +addition to the pre-existing singleton native-Binance-basis lane. All other +reference products remain in bounded batches of at most twelve. The public +batch endpoint, consumer manifest, provider/cache implementation, quotas, +SLA, runtime topology and V1 fallback remain unchanged. The new regression +constructs two strict Mark products, one `2,001ms` Mark product and one +ordinary funding product; it proves singleton isolation exactly at the strict +threshold, complete identity preservation, and ordinary batching for the +looser Mark/funding pair. + +The focused Reference/L2 test module passed `14/14` in `6.563s`; the complete +C2/reference/L2/five-symbol/identity matrix passed `130/130` in `18.414s` +after `py_compile`. Both used the repository Dockerfile builder with source +mounted read-only, `--network none`, UID/GID `10001`, no capabilities, +no-new-privileges and tmpfs-only bytecode/cache state. The temporary builder +tag `qdl-v2-builder:c2-strict-mark-test` is test-only and is retained only +until the final immutable candidate has passed its no-source-mount matrix, +then must be removed in the closure cleanup. No runtime role, V1, Rust, Kafka, +Redis, SQLite, provider, Trading System, alpha, signal, sizing, order path or +market data was changed. The next and only remaining execution is to commit +this source slice, build one immutable Python candidate, roll the same two +query roles with the recorded rollback digest, and run one C2 receipt with its +true 300-second observation. + +**C2 strict-reference immutable image gate (`PASS / QUERY-ONLY ROLLOUT +PENDING`, 2026-09-04).** Commit `19727174f0d31926c28fdc0465637d9cbfb2095e` +was built as `qdl-v2-python:2.0.17-1972717` with immutable digest +`sha256:28d7a6a538b7da69c7718977dae409acaff469fd663f96e09b6b1342d6a0f006`. +OCI revision and release labels are exactly `19727174...` and +`2.0.17-1972717`; the image runs as `qdl:qdl`. Its no-source-mount, +network-disabled, read-only, UID/GID-`10001`, no-capability, +no-new-privileges, tmpfs-only full C2/reference/L2/five-symbol/identity matrix +passed `130/130` in `19.076s` after `py_compile`. The only remaining packet is +the already-approved serial recreate of `query_v2_1` and `query_v2_2` with +their unchanged runtime/TLS/state mounts, health/restart verification, and one +disposable C2 no-order receipt with a true `300s` observation. Any nonzero +returns only those two roles to +`sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`. +V1, Rust, Kafka topology/offsets, Redis, SQLite, ingestors, projectors, +streams, Trading System, alpha and the order path remain excluded. + +**C2 strict-reference runtime receipt (`FAIL-CLOSED / ROLLED BACK`, +2026-09-04).** The approved query-only packet recreated `query_v2_1` then +`query_v2_2` with +`sha256:28d7a6a538b7da69c7718977dae409acaff469fd663f96e09b6b1342d6a0f006`. +Each reached `healthy` with `restart=0`. A new V1 provenance and running +container binding both passed for the frozen `v1.2.4` image +`sha256:dbfb57844977513ae7ec0a4782e04da0213028a789753c6b991f26043b615d65` +at commit `2b0dcf74454c9f87c352d3c47389955aeb955804`. The one detached, +self-removing C2 client ran through the sealed root-to-UID-`10001` bootstrap, +the exact four identities, V2 query/stream endpoints on `executor_network`, +and no Docker socket, provider credential, Kafka/Redis/SQLite, Gateway/Risk or +order authority. + +The receipt exited `1` before its 300-second observation and emitted no +acceptance JSON because the governed `V2 -> V1 -> V2` fallback drill rejected +its V1 response as stale at +`qdl.certification.phase105_fallback._source_age_ms`. This is a real fail-closed +V1 fallback freshness result, not a successful V2 certificate, a strict-Mark +batch error, a V2 query error, or permission to relax freshness/retry the same +receipt. The disposable client self-removed; its compact, payload-free evidence +is under +`/home/bobby/.local/state/qdl-v2/session-liveness-43cdbe3-20260829T162719Z/recovery-2.0.12-8ba4165-20260903/c2-strict-reference-1972717-20260904T090512Z/`. +Per the predeclared nonzero rule, `query_v2_1` then `query_v2_2` were rolled +back serially to +`sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`; +both are `running`, `healthy`, `restart=0`. No other V2 role, V1, Rust, Kafka +topology/offsets, Redis, SQLite, ingestor, projector, stream role, Trading +System, alpha, signal, sizing or order path changed. + +**Decision boundary:** C2 is not certified and the candidate must not be +released. The next permitted work is read-only diagnosis of the exact V1 +fallback product/payload freshness and its mapping. Any source/runtime repair +requires a new, separately journaled scope; do not relax the declared V1 +freshness bound or rerun C2 as a luck-based retry. + +**V1 execution-fallback policy correction (`IN PROGRESS / SAME C2 CLOSURE`, +2026-09-04).** Read-only probing of all twelve manifest-authorized V1 fallback +routes showed the endpoint can currently return fresh Binance trade rows, but +the terminal C2 receipt proved the opposite can occur during a real handoff. +This is expected for a generic last-trade cache: V1 exposes event age but no +typed provider/session liveness, generation or gap state. It therefore cannot +be a trustworthy automatic fallback for the five +`trading-system.paper.stable` execution TRADE products with a `3,000ms` +contract. Waiting/retrying until a print arrives would turn certification into +a luck-based test and would not make V1 semantically equivalent. + +The narrow correction is configuration-only: retain `V2_PRIMARY`, change only +those five V1 fallback declarations to `BLOCKED` with the explicit reason +`V1_EXECUTION_SESSION_LIVENESS_UNPROVEN`, and retain the existing V1 fallback +declarations for monitoring/alpha products whose governed contract permits the +legacy age-only source. No freshness bound is relaxed and no live route is +mutated by this source change. Required source gates must prove exact five-row +selection, unchanged V2-primary identities, retained allowed V1 probes for the +other consumers, and `BLOCKED` routes making zero V1 requests. A fresh C2 +receipt remains a separately bounded runtime action after these gates; it is +not started by this source correction. + +**V1 execution-fallback policy source gate (`PASS / SINGLE C2 CANDIDATE +PENDING`, 2026-09-04).** The stable release-routing revision is now `17`. +Only the five Binance USD-M `TRADE` products of +`trading-system.paper.stable` changed: each remains `V2_PRIMARY` and is now +`fallback: BLOCKED` with +`V1_EXECUTION_SESSION_LIVENESS_UNPROVEN`. This makes the execution-grade +three-second route fail closed if V2 is unavailable rather than silently +switching to a V1 last-trade cache that cannot prove provider/session/gap +state. Monitoring and `alpha.binance.paper.stable` retain their existing, +manifest-governed V1 fallback declarations; all Binance/OKX/VN identities, +freshness bounds, V2 routes, public API/SDK contracts and runtime topology are +unchanged. + +`tests.test_phase105_fallback_acceptance` now proves the exact five-symbol +selection (`BTCUSDT`, `ETHUSDT`, `SOLUSDT`, `DOGEUSDT`, `BNBUSDT`), unchanged +V2-primary route, explicit block reason and zero Trading-System V1 probe; it +also proves the remaining probes are only monitoring/alpha identities. The +focused route/fallback/stream-quota/reference suite passed `55/55` in +`22.276s`. The complete source-mounted C2/reference/L2/five-liquid/identity +matrix passed `138/138` in `20.472s`, network-disabled, read-only, +UID/GID-`10001`, no capability/no-new-privileges and tmpfs-only. No provider, +V1, Rust, Kafka, Redis, SQLite, runtime role, Trading System, alpha, signal, +sizing or order path changed. The next action is exactly one immutable Python +image from this commit followed by the bounded two-query-role C2 packet; any +nonzero rolls back only those two roles to the recorded `bd0163...` image. + +**C2 execution-fallback candidate and receipt (`FAIL-CLOSED / ROLLED BACK`, +2026-09-04).** Commit `cde0fff54df28108aa6568178b23856ed6dcbdf8` was built +once as `qdl-v2-python:2.0.18-cde0fff@sha256:3d21a09314303c1c78d11b272ac09e0456e704af3d54c5f23271e93be2449a83`. +Its OCI revision/release labels, `qdl:qdl` user and the no-source-mount, +network-disabled/read-only `138/138` C2/reference/L2/five-liquid/identity +matrix passed. Only `query_v2_1`, then `query_v2_2`, were serially recreated +with that digest and each reached `healthy`, `restart=0` before acceptance. + +The first disposable client launcher stopped before any bootstrap, identity +copy or endpoint request because its outer `cap-drop ALL` could not read the +bind-mounted bootstrap script. This was a harness permission preflight only, +not a C2 attempt: no C2 output, provider/V1/V2 request, data-plane or order +action existed. The exact documented outer boundary was then used: root with +`no-new-privileges` only for bootstrap, followed by mandatory `setpriv` to UID +`10001` with empty effective/permitted/inheritable/ambient capabilities before +the C2 program started. The payload-free client security receipt records UID +`10001`, `CapPrm/Eff=0` and `NoNewPrivs=1`. + +The single actual C2 ran for about twelve minutes, self-removed, and exited +`1` before the 300-second observation with no acceptance JSON. It failed +closed at the existing signed-cursor no-event handoff requirement for exactly +`alpha.okx.paper.stable`, `OKX.SWAP.PERPETUAL.DOGE-USDT`, `BAR 1h`: +`C2 no-event continuity observation did not confirm the signed cursor stream`. +This is distinct from stream-open quota pacing and the now-blocked V1 +execution fallback; it is not a valid release receipt. Compact failure +evidence is under +`/home/bobby/.local/state/qdl-v2/session-liveness-43cdbe3-20260829T162719Z/recovery-2.0.12-8ba4165-20260903/c2-exec-fallback-blocked-cde0fff-20260904T093057Z/`. +Per the predeclared nonzero rule, only `query_v2_1`, then `query_v2_2`, were +serially restored to +`sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`; +both are `running`, `healthy`, `restart=0`. V1, Rust, Kafka topology/offsets, +Redis, SQLite, ingestors, projectors, streams, Trading System, alpha, provider +state and signal/sizing/order paths were not changed. + +**Decision boundary:** this C2 closure is not certified and the candidate +must not be released. The remaining diagnosis is narrowly limited to the +shared signed-cursor/no-event BAR handoff used by the named OKX `DOGE-USDT` +`1h` route. Do not relax the cursor requirement, freshness SLA or route policy, +and do not rerun C2 as a luck-based retry. A source repair, if the read-only +trace shows one, must retain the existing stream contract and add a direct +regression for quiet-but-connected `1h` BAR handoff before any new bounded C2 +packet. + +**C2 stream-open pacing diagnosis and repair scope (`APPROVED / IN PROGRESS`, +2026-09-04).** Read-only trace proved that the named DOGE route is not missing +provider data or signed-cursor server controls. The C2-only stream pacer is +correctly charging a lazy gRPC `Subscribe` open to the same per-identity quota +as REST. However, the quiet BAR helper starts its fixed two-second *event* +observation on the first `__anext__`; that call includes the client-local +quota wait before the server can emit `REPLAYING` and `LIVE`. The second quiet +session can therefore time out with an empty control list while waiting for its +own admitted stream-open slot. This is a C2 harness timing defect introduced +by the approved quota pacing, not an OKX data or cursor-contract defect. + +The repair remains deliberately narrow: C2 will distinguish bounded local +stream-open scheduling from the existing two-second post-open quiet-event +window. It will wait only to the existing C2 opening deadline for the first +server control, then retain the original signed `REPLAYING -> LIVE` requirement +and original final/current BAR validation. Freshness, provider/session quality, +cursor semantics, manifest policy, public SDK/runtime contracts and all data +plane roles remain unchanged. Regression must prove a quota-delayed open can +receive both controls, a connection that emits one/no control still fails on +the original post-open bound, and the stream pacer still charges REST plus +every open/reconnect to one identity quota. Source gates must pass before one +new immutable Python image and exactly one 300-second C2 retry; the existing +two-query-role rollback remains unchanged. + +**C2 paced-stream-open source gate (`PASS / IMMUTABLE BUILD PENDING`, +2026-09-04).** The shared receipt helper now accepts an optional C2-only +`stream_open_timeout_seconds`. With no value, its prior behavior is byte-for- +byte equivalent: no other receipt path changes. With the C2 value, it permits +only the bounded local admission wait until the first server response. After +`REPLAYING` (or any server handshake control), the original event window is +restarted; a client-local `RECONNECTED` or `SNAPSHOT_REPLACED` opens one new +bounded admission window for the next lazy subscription. The phase-10.5 C2 +runner supplies its existing global opening budget, which is already bounded +by the enclosing `900s` opening gate. It does not alter any provider or public +SDK timeout/SLA. + +The isolated source-only regression ran as UID/GID `10001`, read-only, +network-disabled, capability-free/no-new-privileges with tmpfs scratch: +`python -m unittest tests.test_phase103_consumer_receipt_harness +tests.test_phase105_identity_acceptance tests.test_phase105_fallback_acceptance` +passed `77/77` in `19.588s`. New cases prove a quota-delayed quiet final BAR +receives `REPLAYING -> LIVE` before the original post-open observation, an +incomplete `REPLAYING`-only handshake remains rejected before any current read, +and the phase-10.3 certificate function receives the bounded opening budget. +Existing C2 stream/REST shared-quota and fail-closed open regressions remain +green. `git diff --check` passed. No runtime service, image, provider, Kafka, +Redis, SQLite, V1, Trading System, alpha or order path changed. Next: build one +immutable Python candidate, run the complete no-source-mount source matrix, +then serially recreate only the two approved query roles for one C2 retry. + +**C2 paced-stream-open immutable candidate (`PASS / QUERY-ONLY ROLLOUT +PENDING`, 2026-09-04).** Commit `7d7f8102f6f59e4e7b7a444c7b26dc9faa65cddd` +was built once as `qdl-v2-python:2.0.19-7d7f810` with immutable digest +`sha256:2a087564eb4d442a59e6eff05629934ef221ad0b6b448cfc3c9c110578d88b6f`. +OCI source revision and release labels match the commit/tag and the process +user remains `qdl:qdl`. The candidate was exercised without a source mount, +with network disabled, a read-only root filesystem, UID/GID `10001`, no Linux +capabilities, `no-new-privileges`, and tmpfs-only scratch. `py_compile` plus +the C2 receipt harness (`50/50`), identity/fallback suites (`27/27`), +reference batch (`26/26`), reference-v2 (`16/16`), reference/L2 consumer +acceptance (`14/14`) and five-liquid handoff (`8/8`) passed: `141/141` total. +No runtime role, provider request, durable store, V1 route, Trading System, +alpha, signal, sizing or order path changed during the gate. + +**Authorized next action and rollback.** Serially recreate only +`query_v2_1`, verify `healthy` with `restart=0`, then `query_v2_2` under their +unchanged runtime, TLS and state mounts using the candidate digest. Run exactly +one disposable four-identity C2 no-order receipt with its genuine +`300s` observation. A nonzero receipt restores only those same query roles, +serially, to +`sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`. +V1, Rust, Kafka topology/offsets, Redis, SQLite, ingestors, projectors, +streams, Trading System, alpha and all order/signal/sizing paths remain out of +scope. Candidate/test-only artifacts will be inventory-cleaned only after this +single terminal receipt, retaining the active image and named rollback image. + +**C2 paced-stream-open runtime receipt (`FAIL-CLOSED / ROLLED BACK`, +2026-09-04).** Two launcher-only preflights stopped before the C2 program: +the immutable image's declared non-root user needed an explicit root bootstrap, +and the next launch mounted the recovery subdirectory rather than the existing +read-only `qdl-v2` state root. Neither preflight copied identities, opened an +endpoint, created C2 output, or consumed the approved receipt. The clean third +launcher used root only for the established tmpfs copy/bootstrap and then +executed the client as UID `10001`, with empty effective/permitted/inheritable +and ambient capabilities and `NoNewPrivs=1`. + +The one actual disposable C2 client then exited nonzero before its true +`300s` observation. It had already passed the repaired paced stream-open path; +the terminal, unrelated fail-closed condition was +`alpha.okx.paper.stable / OKX.SWAP.PERPETUAL.SOL-USDT / BOOK_SNAPSHOT`: the +secondary V2 read could not prove a current complete, gap-free snapshot inside +the declared freshness policy. The client wrote no acceptance receipt, had no +order/provider credential, Docker socket, Kafka/Redis/SQLite mount or +Gateway/Risk/alpha authority, and self-removed. Compact payload-free evidence +is retained under +`/home/bobby/.local/state/qdl-v2/session-liveness-43cdbe3-20260829T162719Z/recovery-2.0.12-8ba4165-20260903/c2-paced-open-7d7f810-r3-sQFo1t/`. + +Per the approved nonzero rule, only `query_v2_1`, then `query_v2_2`, were +restored serially to +`sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`. +Both are `running`, `healthy`, `restart=0`. V1, Rust, Kafka topology/offsets, +Redis, SQLite, ingestors, projectors, streams, Trading System, alpha, market +data, signals, sizing and order paths were unchanged. + +**Decision boundary.** The C2 pace repair is source-tested, but this receipt +does not certify release. The SOL L2 completeness/freshness failure is outside +the approved C2 scheduling scope and must be separately diagnosed and repaired +at the shared L2 materialization/quality boundary with a targeted regression +before any new C2 packet. Do not relax freshness/completeness, silently fall +back, or rerun C2 as a luck-based retry. The unused candidate and builder may +be removed after their hashes/evidence are retained; scoped failure evidence +and the active named rollback image remain. + +**Scoped artifact cleanup (`COMPLETE`, 2026-09-04).** After the terminal +receipt and rollback verification, only unused Data Layer test/candidate images +were removed: `qdl-v2-python:2.0.13-f2e5654` through +`qdl-v2-python:2.0.19-7d7f810`, plus +`qdl-v2-builder:c2-strict-mark-test`. The active +`qdl-v2-python:2.0.12-8ba4165@sha256:bd0163...` rollback/runtime image, +running containers, volumes, networks and compact evidence directories remain. +Root disk changed from `164G used / 126G available` to `160G used / 130G +available`; Docker image storage changed from `17.21GB` to `10.62GB`. +BuildKit reports `11.88GB` reclaimable cache, but it is shared/unattributed at +this point and was deliberately not broadly pruned. Both rolled-back query +roles remained `healthy`, `restart=0` after cleanup. + +### L2 execution-readiness closure (`APPROVED / IN PROGRESS`, 2026-09-04) + +**Goal.** Close the real multi-symbol execution-L2 gap revealed by the paced +C2 receipt, then run exactly one new no-order C2 certificate. The applicable +source of truth is the declared execution demand, not a manually maintained +symbol allow-list: every currently active execution-grade `BOOK_SNAPSHOT` / +`BOOK_DELTA` pair for Binance USD-M and OKX Swap must receive the same shared +Rust-core verified-view materialization policy. Current intended scope is +`BTC`, `ETH`, `SOL`, `DOGE`, and `BNB` on both venues, with depth `100`. + +**Known defect and invariant.** The active three-core runtime proves that the +previous refresh tool hard-coded two ETH source IDs. All other active L2 pairs +therefore fell back to `snapshot_refresh_seconds=30` instead of the declared +hot materialization cadence. This closure must remove that hard-coded source +set. Provider bootstrap/renewal remains `30s`; only a *verified, gap-free* +canonical book view may materialize at the common hot cadence. A duplicate, +out-of-order frame, gap, resync, unverified sequence, wrong identity, or +cross-replica divergence must remain non-execution-eligible. No freshness SLA +may be relaxed and no synthetic book, V1 substitution, per-symbol process, new +topic, or topology may be used to force a pass. + +**Approved scope and rollback.** First run a read-only typed-status matrix for +the ten active venue/symbol books through both V2 query replicas. The matrix +must record compact payload-free `state`, event age, provider-session +liveness, `complete`, `gap_open`, book generation, sequence-verification, +source sequence, materialization age and watermark/replica identity. Then +make the runtime compiler manifest-derived, add deterministic Rust/Python +regressions and make C2 retain that same compact typed status on failure. +After source gates, regenerate only `core.json`, `core-002.json` and +`core-003.json`, serially recreate only `rust_core`, `rust_core_2` and +`rust_core_3`, perform a short real-provider L2 readiness preflight, then run +one disposable four-identity C2 no-order receipt with its real `300s` +observation. A nonzero core/preflight/C2 result restores exactly the three +core JSON files and their existing image selector from the generated rollback +directory. V1, Kafka topology/offsets, Redis flushes, SQLite deletion, +ingestors, projectors, query/stream roles, Trading System, alpha runtime, +signals, sizing and all order paths are excluded. + +**Required evidence and exit.** Source tests must cover all ten execution +books across both replicas for verified ready, quiet-but-connected, +duplicate, out-of-order, gap/resync and recovery behavior; compiler tests must +prove adding/removing a declared execution book changes the derived set without +hard-coded symbols. The real preflight must show both replicas `LIVE`, +`complete=true`, `gap_open=false`, verified sequence and bounded snapshot age +for all ten books before C2 starts. C2 then proves the existing final-BAR, +quote/trade, reference, signed-cursor/reconnect and governed V1 fallback +behaviors without an order or consumer-state mutation. Only a passing C2 is a +release-certification exit. Each slice records exact commands, counts, runtime +image/config hashes, rollback state and cleanup evidence below this entry. + +**Source compiler and diagnostic slice (`PASS / RUNTIME PREFLIGHT PENDING`, +2026-09-04).** Replaced the hand-maintained ETH-only hot-book allow-list in +both Rust-core refresh tools with `qdl.runtime.execution_l2`, a fail-closed +join of `stable-crypto-demand.yaml`, catalog and acquisition contracts. It +derives the ten active execution physical source IDs (`BTC/ETH/SOL/DOGE/BNB` +on Binance USD-M and OKX Swap) only when each has an exact +`BOOK_SNAPSHOT`/`BOOK_DELTA` pair, depth `100`, Rust-native contiguous +acquisition, provider refresh `30s`, and common verified-view materialization +of `1000ms`. The acquisition configuration now declares that common cadence +for all ten pairs; provider refresh itself remains unchanged at `30s` and +ingestors do not gain a per-symbol loop or a new service. + +The C2 strict-snapshot harness now attaches bounded, payload-free typed status +evidence to a non-retryable stale BOOK failure: identity, feed, state, event +age, provider-session liveness, `complete`, `gap_open`, execution eligibility, +policy and at most sixteen flags. Phase 10.5 renders that evidence as a compact +failure JSON with primary/secondary replica attribution; it records no book +levels/prices, cursor, credentials or market payload. + +Source gates actually run inside immutable +`qdl-v2-python:2.0.12-8ba4165@sha256:bd0163...`, UID/GID `10001`, read-only +root, `--network none`, no capabilities, no-new-privileges and tmpfs-only +scratch: + +- `tests.test_execution_l2_materialization`, + `tests.test_refresh_v2_rust_core_runtime`, and + `tests.test_refresh_v2_l2_core_runtime`: **15/15 PASS** in `12.866s`. +- `StableDeploymentContractTests.test_hot_l2_materialization_is_core_only_and_keeps_provider_refresh`, + all `Phase103QuietQuoteRetryTests`, and all + `Phase105IdentityAcceptanceTests`: **29/29 PASS** in `1.498s` after one + test-fixture correction (`subscription_id`, not a runtime-only binding ID). + +The compiler tests prove an incomplete snapshot/delta pair and missing hot +cadence fail closed; refresh tests prove only the three core JSON files and +optional existing immutable image selector can change, with exact rollback +copies. No runtime role, image, provider call, Kafka/Redis/SQLite state, V1, +Trading System, alpha, signal, sizing or order path has changed. Next approved +slice: run the new read-only ten-book/two-replica matrix, then only if it +passes apply/roll exactly `core.json`, `core-002.json`, `core-003.json` and +`rust_core`, `rust_core_2`, `rust_core_3` before the one C2 receipt. + +**Matrix hardening and final source gates (`PASS / RUNTIME PREFLIGHT PENDING`, +2026-09-04).** `scripts/phase105_execution_l2_status_matrix.py` now derives +the exact ten physical `BOOK_SNAPSHOT` products from the declared execution +demand and reads each through both V2 query replicas. Its compact, payload-free +row records product/replica identity, typed quality state, event age, +provider-session liveness, `complete`, `gap_open`, execution eligibility, +verified book generation/native sequence/depth, materialization age and +durable `watermark_offset`; no levels, prices, quantities, cursor or credential +is retained. A status-transport failure itself becomes a typed fail-closed row +instead of an unstructured harness error. + +The final source-only Python matrix/regression ran `118/118 PASS` in +`42.368s` with exit `0`, network disabled, read-only root, UID/GID `10001`, no +Linux capabilities, no-new-privileges, `1 CPU`, `768MiB` memory and no provider +or runtime access. The visible BAR/DNSE messages are intentional failure +fixtures that passed their fail-closed assertions. The Rust workspace library +suite ran release-mode from a temporary local builder with network disabled, +`1.5 CPU` and `2GiB` memory; it exited `0`, including Binance/OKX verified +materialization, gap/resync, duplicate suppression and cross-source isolation. +One explicitly isolated Redis coordination test remained `ignored` because no +test Redis URL was supplied. The temporary Rust test container and +`qdl-l2-rust-test:8bd84de` image were removed immediately after the gate. + +**Runtime preflight and bounded recovery (`ROLLBACK PASS / ONE FINAL ATTEMPT +PENDING`, 2026-09-04).** Immutable source tool image +`qdl-v2-python:2.0.12-1d0110e-l2-tool@sha256:198ff9...` first read the ten +books through both V2 query replicas: `10/10` parity passed, with no provider +connection, order action, cursor retention, market levels or prices recorded. +The refresh compiler then wrote exactly `core.json`, `core-002.json` and +`core-003.json`, plus reconciled `rollout.env` to the Rust digest actually +running (`sha256:d86f0e...`, revision `8ba4165`). Its former selector pointed +to absent digest `sha256:c63d54...`; retaining that stale selector would make a +Compose recreate pull a non-existent rollback binary. No Rust binary changed. + +The three Rust cores were recreated serially and all remained `running`, +`restart=0`, `OOMKilled=false`. An immediate post-roll matrix at about eight +seconds failed only `BINANCE/USDM DOGEUSDT`: both replicas reported +`STALE`, age about `61s`, `complete=true`, `gap_open=false`, and no verified +snapshot. The other nine books remained `LIVE`, complete, gap-free, +depth-100 and sequence-verified. This was not retried: the three core JSON +files were restored from their exact SHA-verified rollback copies, the +selector was kept at the actual unchanged `d86f0e...` digest so the rollback +could be recreated, and all three cores were rolled back serially. The recovery +matrix passed `10/10` again. + +**Diagnosis and final decision boundary.** Rust L2 adapter state is +intentionally memory-resident and fail-closed after a core restart. A Binance +diff-depth book cannot become execution-eligible until the next provider REST +snapshot bridges its new state; the declared common provider bootstrap/renewal +contract is `30s`. The failed eight-second probe was below that documented +bootstrap bound, not evidence that relaxed freshness or a synthetic snapshot +is needed. The final approved attempt therefore waits a deterministic `40s` +after the last of the three serial core restarts (`30s` provider bound plus +`10s` scheduling margin), then runs exactly one ten-book/two-replica matrix. +Only `PASS` permits exactly one C2 no-order `300s` receipt. A matrix/C2 failure +again restores the three JSON files and retains the actual immutable `d86f0e...` +binary selector; no C2 retry or SLA relaxation is allowed without a new source +diagnosis. + +**C2 closing-read diagnosis and bounded repair (`APPROVED / IN PROGRESS`, +2026-09-04).** The post-preflight matrix completed `10/10` execution books on +both query replicas after the declared `40s` Rust-book bootstrap bound. The +single disposable C2 then failed before writing its receipt inside closing +`warmup:batch` with an HTTP read timeout. This is not an L2 quality failure: +the C2 scope has `299` V2-primary products, including `150` durable/pass-through +BAR products. The opening proof already reads each bounded BAR history and +proves signed cursor/reconnect. Closing incorrectly rebuilt the same +`_c2_requirement` for every BAR (`min(manifest rows, 700, retained capacity)`) +and sent those histories concurrently to both replicas for all four identities. +That turns a current-state closing check into a large duplicate history transfer +and can exceed the unchanged per-request `15s` transport deadline. + +The final approved core attempt did wait the deterministic `40s`: its +ten-book/two-replica matrix passed `10/10` before the one actual C2 client +started. C2 then reached closing revalidation and exited on that unwrapped +`httpx.ReadTimeout`; it produced no acceptance receipt and is not a +certificate. The exact three core JSON files were restored from the generated +rollback directory and the three cores were recreated serially with the +unchanged `d86f0e...` image selector. This returned the runtime to its prior +baseline without touching V1, query/stream, Kafka/Redis/SQLite, Trading +System, alpha or order state. + +The approved repair is limited to the C2 harness, not the public SDK/query +contract: opening warmup, signed cursor, reconnect, full history/finality and +all manifest policy remain unchanged. Closing retains the exact product +identity, grade, source policy, interval, freshness/session/gap/finality, +coverage, recovery and deadline fields, but requests exactly one final BAR row +for BAR products. It continues to use `require_all=true`, validates that row +strictly, and requires exact final-BAR content parity between the two replicas. +Non-BAR closing reads retain their existing current-view semantics. The patch +must also convert a closing batch transport failure into compact payload-free +typed evidence (consumer, replica, batch identity digest/count and bounded +feed-status observations); it must never record a price, level, quantity, +cursor, credential or raw provider payload. + +**Exit / rollback.** Add deterministic regressions for one-row-only closing +BAR requests, preserved public/full opening requirements, strict final-BAR +parity, and compact timeout evidence. Run the existing source-only C2/L2 +matrix under the constrained immutable image. No query/core/ingestor/runtime +service is changed by this harness repair. Only after those source gates pass, +reapply the already-approved three core JSON files, serially recreate only +`rust_core`, `rust_core_2` and `rust_core_3`, wait the documented `40s`, rerun +the ten-book matrix, and consume exactly one new `300s` C2 receipt. Any +nonzero result restores only those three core JSON files and core roles; V1, +Kafka offsets/topology, Redis, SQLite, ingestors, projectors, query/stream, +Trading System, alpha and every order path remain excluded. + +**C2 closing-read source gate (`PASS / RUNTIME C2 CANDIDATE PENDING`, +2026-09-04).** The harness now derives a separate closing requirement. For a +BAR it retains the exact manifested identity, grade, source policy, interval, +freshness/session/gap/finality/coverage/recovery controls and all warmup +deadline/cache fields, while changing only the history horizon to one final +row. Both supported SDK representations are covered: an explicit +`WarmupSpecification(rows=N)` and the equivalent `warmup_limit=N` form. The +opening C2 proof remains unchanged at its bounded historical window and signed +cursor/reconnect proof; closing compares the one final BAR's canonical content +across both replicas. A failed closing transport batch now emits +`qdl.phase105.c2-closing-batch-failure.v1` with a batch identity digest/count +and at most one payload-free typed status per feed. It never records market +payload, levels, prices, quantities, cursors or secrets. + +The real sealed scope was compiled source-only: `234` durable/pass-through +stream products, including `150` BAR products, all resulting in exactly +`rows=1` with `require_final_bars=true` and `require_full_coverage=true`. +The constrained immutable tool image +`qdl-v2-python:2.0.12-1d0110e-l2-tool`, with the candidate source mounted +read-only, ran the execution-L2, C2 receipt, +consumer/route/fallback, reference/L2, universe and core-refresh regression +matrix: **`175/175 PASS` in `78.422s`**, network disabled, read-only root, +UID/GID `1001`, no Linux capabilities, `1 CPU`, `768MiB`, tmpfs-only scratch +and no runtime/provider access. No runtime role, image selector, provider +session, Kafka/Redis/SQLite/V1 state, Trading System, alpha, signal, sizing or +order path changed by this source gate. + +**Next exact packet.** Build one immutable Python C2-client image from this +source revision, apply the already-reviewed manifest-derived materialization to +only `core.json`, `core-002.json`, `core-003.json`, serially recreate only the +three existing Rust cores, wait `40s`, and require the ten-book/two-replica +matrix to pass. Then run exactly one disposable four-identity no-order C2 with +its full `300s` observation. Any nonzero result restores the three core JSON +files/core roles; the Python candidate is test-only and does not alter query +or stream roles. Cleanup retains only the active image set and named rollback +image after the terminal result. + +**L2 runtime recovery fact and revised terminal packet (`IN PROGRESS`, +2026-09-04).** The manifest-derived core refresh was applied to exactly +`core.json`, `core-002.json`, and `core-003.json` (new SHA-256 values +`7a092e...f7dfb381`, `05ba9e...02be6edf`, and +`e91c038b...de0d1698`), then `rust_core`, `rust_core_2`, and `rust_core_3` +were recreated serially on their unchanged +`sha256:d86f0e...ed86983` Rust image. All three were `running`, +`restart=0`, and `OOMKilled=false` after the documented 40-second provider +bootstrap window. The source regression matrix ran `175/175 PASS` both from a +constrained source mount and again from immutable client image +`qdl-v2-python:2.0.12-0843d2d-c2@sha256:b17173...32f42e1` with no source +mount. + +The live L2 matrix then exposed an operational recovery defect before C2: both +existing query replicas had previously been kernel-OOM killed at 12:30 UTC +(`exit=137`, `OOMKilled=true`, `restart=no`) and their prior Python image ID +`sha256:bd0163...` was already pruned from the local image inventory. Starting +the stopped containers is possible, but recreating them through Compose fails +because the runtime selector points at a non-retained image ID. Once started, +the typed ten-book matrix reached both replicas but correctly failed all ten +books as `STALE` with `complete=true` and `gap_open=false`; Rust core progress +and the 30-second provider snapshot renewals continued. This is a +projection/query recovery failure, not a DOGE/SOL-specific source admission +failure and not grounds to relax freshness. + +**Revised bounded terminal scope.** Build one canonical immutable shared +Python image from source revision `0843d2d` using a canonical +`qdl-v2-python:2.0.12-0843d2d` tag and matching OCI revision/version labels. +Seal its digest in the runtime selector. Roll only the seven existing V2 +Python cache/read roles, serially and with their existing TLS/state/runtime +mounts: `stream_v2_passive`, `stream_v2_active`, `projector_v2`, +`projector_v2_2`, `projector_v2_3`, `query_v2_2`, `query_v2_1`. The stream +pair must keep one healthy lease holder; projectors must prove cache binding +and canonical consumption before query replicas are rolled. This adds no +service, container, topic, symbol worker, provider credential, data-plane +schema, Redis flush, SQLite deletion, Kafka offset change, V1 mutation, +Trading System/alpha change, or order action. It replaces an unrecreatable +pruned Python runtime with one shared immutable image and reloads the same +manifest/catalog contract used by the cores. + +**Rollback and exit.** Retain the currently running legacy Python containers +only until the new image's seven-role recovery proves ready; record their +per-role image IDs and runtime mounts before each recreate. If any role fails +its bounded health/lease/cache gate, stop immediately and restore only that +role to its recorded container/image when locally available; otherwise retain +V1 as the product fallback and fail closed rather than invent an image. +After all seven roles are healthy, wait through one provider renewal window, +require the compact ten-book/two-replica L2 matrix to pass, then run exactly +one C2 no-order 300-second receipt. C2 failure records compact typed evidence +and does not trigger additional retries. A pass permits cleanup of the +disposable C2 client/test artifacts while retaining the active canonical image +and one explicitly named rollback image. This revision supersedes the prior +statement that query/stream/projector could remain excluded: their actual +pruned-image/projection state makes that exclusion incompatible with an honest +C2 certificate. + +**L2 stale-envelope correction (`APPROVED / IN PROGRESS`, 2026-09-04).** The +seven Python cache/read roles above have now recovered on the single canonical +immutable image `qdl-v2-python:2.0.12-0843d2d`; both query replicas are +healthy. The resulting typed ten-book/two-replica matrix still correctly +fails closed because each latest execution L2 view carries a `received_at_ns` +roughly 45 minutes older than its current spool commit, while the raw Kafka +topic and all native-ingestor/core clocks are current. This is neither a +SOL/DOGE-specific contract issue nor a reason to relax query freshness. + +**Approved narrow repair.** Rust is the authority for this provider-neutral +ingress invariant. For any declared L2 binding, core must reject a raw frame +whose receipt age exceeds that binding's existing +`snapshot_refresh_seconds`; it must request adapter resync and emit ordinary +bounded quarantine evidence, never re-date or materialize the old frame. The +native lossless publisher must use the same declared bound for BOOK delivery: +a Kafka delivery that cannot complete before the provider renewal bound ends +the affected connection generation so its normal bootstrap/reconnect path +obtains a fresh verified view. Python remains unchanged. This is shared +Binance/OKX behavior, adds no public schema, service, worker, topic, cache or +SLA relaxation. + +**Gate / runtime packet / rollback.** Add only deterministic Rust unit tests +for stale-L2 rejection/resync and delivery-bound calculation, then run the +existing targeted Rust suites in the constrained builder. Build one canonical +Rust image from the tested commit; serially recreate exactly `rust_core`, +`rust_core_2`, `rust_core_3`, then `ingestor_binance_usdm` and +`ingestor_okx_swap`. Wait one 40-second bootstrap bound, require the compact +ten-book/two-replica matrix to pass, then run exactly one 300-second C2 +no-order receipt. V1, Kafka topology/offsets, Redis, SQLite, projectors, +query/stream, Trading System, alpha and order paths remain untouched. +Rollback restores only these five roles to the current +`sha256:d86f0e832ba945d302fd3f782e26fd41c5b08709a80f6de16bdd36af5ed86983` +image and their saved runtime configs. A final C2 pass permits only scoped +image/BuildKit cleanup, retaining active images plus one named rollback image. + +**Build-capacity guard.** The first disposable Rust builder attempt was killed +before it could tag an image while compiling the whole release binary set; it +did not alter any running role. The release Dockerfile now constrains Cargo to +one job, so the canonical Rust build is deterministic within the host's +available memory rather than relying on unconstrained parallel compilation. +This is build hygiene only: it changes no runtime thread, provider cadence, +data-plane contract or topology. The temporary builder tag is retained only +until the targeted source gate finishes, then removed with the final scoped +image/cache cleanup. + +**Source gate (`PASS`, 2026-09-04).** The constrained builder ran with one +CPU and a 4 GiB memory cap, network disabled, and no provider/runtime access. +`cargo fmt --all -- --check` passed against the read-only worktree. The two +new release-profile regressions passed: `stale_l2_raw_is_quarantined_without_redating_and_fresh_snapshot_recovers` +and `book_delivery_bound_is_exact_and_never_applies_to_non_book_frames`. +The first fixture run exposed only two test mistakes (escaped JSON, then the +documented Binance delta-to-snapshot bridge requirement); both were corrected +before the passing run. No runtime container, broker, cache, topic, offset, +consumer, provider request, V1 route, Trading System, alpha or order state was +changed by the source gate. The next operation is the already-approved single +canonical Rust image build and five-role rolling packet; there will be no +additional test image or topology. + +The named regression container `qdl-rust-regression-l2-temp` exited `0` with +`OOMKilled=false` and was removed immediately. Its temporary builder image is +the only remaining test artifact and is retained solely to build the canonical +runtime image in the next step. + +**Stale-envelope runtime rollout and catch-up evidence (`IN PROGRESS / C2 +HELD`, 2026-09-04).** One canonical Rust runtime image was built from tested +source `d53609850558bd6e243ada220c93a9500ef48757` with OCI revision +`d536098` and version `2.0.12-d536098`: +`sha256:36a822c0ef61fb122dbf8fa12221cff27ad6a863976424be1407cd345f4dce65`. +Before mutation, the operator state received an exact SHA-verified rollback +copy of `rollout.env`, `core.json`, `core-002.json`, `core-003.json`, +`ingestor-binance-usdm.json`, and `ingestor-okx-swap.json` under +`rollback-d536098-20260904T150845Z`; the prior image is retained explicitly as +`qdl-v2-rust:rollback-2.0.12-8ba4165@sha256:d86f0e...ed86983`. + +Only the approved existing roles were serially recreated: +`rust_core`, `rust_core_2`, `rust_core_3`, `ingestor_binance_usdm`, and +`ingestor_okx_swap`. Each started on `36a822...dce65` with `restart=0` and +`OOMKilled=false`. V1, Kafka topology and offsets, Redis, SQLite, +projector/query/stream roles, Trading System, alpha, and order paths were not +changed. The first disposable L2 preflight exposed a malformed legacy C2 +identity-directory mount before any request. It was deleted; the replacement +uses four individual read-only governed identity file mounts, tmpfs-only +cursor state, UID/GID `10001`, no capabilities, no provider credential and +only the two V2 query replicas. + +The replacement real V2 preflight completed and retained compact typed +evidence only (`10` physical execution books x `2` replicas, +`provider_connections=0`, `order_actions=0`, no cursor or payload retained). +It correctly returned `FAIL`, not a certificate: every view remains +`STALE`, `complete=true`, `gap_open=false`, `execution_eligible=false`, with +an approximately 46-minute event age. Read-only Kafka evidence isolates the +cause: the Rust core group is near current (partition lag `69..266`), while +`stable-projector-v1` has an inherited canonical backlog (`1,434,640` records +at the last measurement) from its earlier recovery. This is a normal durable +catch-up, not a symbol-specific L2 parsing defect. Its initial observed drain +rate was about `50k` records/minute. No offset reset is permitted because it +would silently skip retained BAR/history records. C2 is deliberately held +until projector lag is drained and the ten-book matrix returns `PASS`; no C2 +attempt has been counted in this packet. + +**Scoped artifact cleanup (`PASS / RUNTIME CATCH-UP CONTINUES`).** After the +canonical image completed, removed only unreferenced test artifacts: +`qdl-v2-rust-builder:2.0.12-l2-stale-test`, the two disposable Python C2/L2 +tool images, and the obsolete stream/projector rollback tag. The redundant old +Rust release tag was removed while its explicit rollback tag was retained. An +attempt to remove `qdl-v2-python:2.0.12-8ba4165` was refused because a stopped +container still references it; it was left intact and not force-removed. +`docker builder prune -f` removed only unused BuildKit cache. Docker images +fell from `27 / 16.12GB` to `23 / 11.7GB`; BuildKit from `18.34GB` (`15.39GB` +reclaimable) to `2.278GB` (`0B` reclaimable); root filesystem changed from +`177GB used / 113GB available` to `161GB used / 130GB available`. No volume, +network, source, runtime-state, active container, Kafka/Redis/SQLite data or +V1 artifact was removed. Retained set is active V2 Python +`f608105...b3014`, active Rust `36a822...dce65`, one Python rollback +`86a236...cbf5d`, one Rust rollback `d86f0e...ed86983`, and V1 fallback +`dbfb57...15d65`. + +**Next and only release action.** Allow the existing projector group to catch +up without offset mutation; when its lag is near zero, run one new ten-book +matrix using the corrected file-bind client. Only a `PASS` may start the one +already-approved 300-second four-identity C2 no-order receipt. A non-pass +returns to the exact five-role rollback above; it does not create another +phase, image, topology, or retry ceremony. + +**Stable stream deadline-propagation repair (`IN PROGRESS / NARROW RELEASE +BLOCKER`, 2026-09-04).** Read-only runtime inspection found the exact reason +the inherited projector catch-up intermittently stalls: the active containers +declare `QDL_STABLE_REQUEST_DEADLINE_SECONDS=90`, and the query application +propagates that value, but `create_stable_stream_runtime()` constructs its +`BoundedRequestMiddleware` without `request_deadline_seconds`. The stream +therefore silently falls back to the `RequestBounds` default of `10s`, emits +truthful `504 DEADLINE_EXCEEDED` responses while a canonical batch is still +being durably accepted, and forces projector generation reconnects. This is a +configuration-propagation bug, not an L2/provider/data-quality defect. + +Approved scope is deliberately limited to one shared stable-request-bounds +constructor, use by the existing query and stream applications, and a +regression proving a configured `90s` deadline reaches both. No public API, +schema, deadline value, Kafka topic/offset, Redis/SQLite state, provider, +Rust role, V1 path, Trading System, alpha, or order path changes. Source gates +are targeted unit/regression tests in the existing constrained Python image. +If they pass, build one immutable Python image and serially recreate only +`stream_v2_active` then `stream_v2_passive`, retaining their exact current +image/runtime selector as rollback. The projectors continue from their +committed offsets without reset; after their natural catch-up, rerun one +ten-book matrix and the already-authorized single C2 no-order receipt. Any +failure restores only the two Stream roles and does not create a retry image, +new role, or topology. + +**Source gate (`PASS / STREAM ROLLOUT PENDING`, 2026-09-04).** +`qdl.runtime.stable.stable_request_bounds()` is now the single constructor for +both the V2 query application and internal stable-stream ingress. It carries +the configured maximum request bytes, request deadline and concurrency bound, +so a declared `QDL_STABLE_REQUEST_DEADLINE_SECONDS=90` cannot be applied to +query while silently omitted from stream. The focused boundary regression and +the complete stable-edge module passed in the existing immutable Python image +with network disabled, read-only root, UID/GID `10001`, and tmpfs-only scratch: +`1/1 PASS` for the direct propagation case and `52 PASS, 1 skipped` in +`20.670s` for `tests.test_phaseb_stable_edge`. No provider call, runtime role, +image selector, Kafka/Redis/SQLite state, V1 route, Trading System, alpha or +order mutation occurred during source verification. + +The next bounded mutation is one immutable Python image from this tested +commit, an exact runtime backup, then serial recreation of only +`stream_v2_active` and `stream_v2_passive`. Existing projectors keep their +committed offsets and will naturally drain; no offset reset is valid. Retain +the currently active Python image as the sole rollback selector until the +matrix and C2 receipt pass. + +**Stream rollout and recovery measurement (`PASS / C2 HELD FOR DURABLE +CATCH-UP`, 2026-09-04).** One immutable Python image was built from source +`8343f10cb483cc78c66763270957aed2c9d23b35` as +`qdl-v2-python:2.0.12-8343f10@sha256:08dd22c37c3b4373622d4a1315897343c541ad877ed24ab2589db386808ed29b`. +Before selector change, exact `rollout.env` rollback evidence was SHA-recorded +under `rollback-stream-deadline-20260904T154113Z`; the active prior image is +retained as `qdl-v2-python:rollback-stream-0843d2d@sha256:f608105...b3014`. +Only `stream_v2_active` and then `stream_v2_passive` were serially recreated. +Both reached `running/healthy`, `restart=0`, `OOMKilled=false`. No projector, +Rust, ingestor, query, V1, Kafka topology/offset, Redis/SQLite, Trading +System, alpha or order role changed. + +A bounded post-rollout recovery measurement recorded projector group lag +`1,107,501 -> 1,091,748 -> 1,075,713` in `79s`; the sampled recent +stream-side `504` count was zero after rollout. This verifies the hitherto +silent `10s` middleware deadline was the catch-up stall, while preserving +durable ordering rather than skipping retained data. C2 remains intentionally +held until lag is near zero and the compact ten-book/two-replica matrix passes. +No additional image, role, retry or topology is permitted before that gate. + +**Bounded projector catch-up tuning (`IN PROGRESS / SAME RELEASE CLOSURE`, +2026-09-04).** After the stream deadline correction, all `504` samples stayed +at zero and lag drained, but read-only partition evidence showed nearly all +remaining work assigned to one ordered canonical partition while its projector +was only about half of its existing `0.75` CPU allowance. The stable projector +already validates `QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS` through `512`; the +canonical Compose value is an unnecessarily conservative `128`, while the +existing `8MiB` per-batch byte cap, `2,048` pending-record cap, `32MiB` +pending-byte cap, request-byte bound and idempotent/checkpoint ordering remain +unchanged. The narrow correction raises only the declared projector batch +count to `512` for the three existing replicas. + +This is throughput-only: it does not alter source/provider events, sequence, +event IDs, durable ACK/checkpoint ordering, retention, Kafka partitions or +offsets, Redis/SQLite state, public contracts, image binary, V1, Rust, +ingestors, query/stream, Trading System, alpha or order paths. Source gate: +update the Compose regression and prove `512` is accepted while `2049` +continues to fail. Runtime packet: SHA-backup the exact Compose file, serially +recreate only `projector_v2`, `projector_v2_2`, `projector_v2_3` on their +current immutable Python image with unchanged runtime/TLS/state mounts; then +measure lag, error count, health/restart/OOM. Rollback restores the saved +Compose file and recreates only those same three projectors at `128`. C2 stays +held until durable lag is near zero. + +**Source gate (`PASS / PROJECTOR ROLLING PENDING`, 2026-09-04).** The canonical +Compose declaration now sets the existing three projector roles to the already +validated `512` record bound. The constrained immutable Python image +`qdl-v2-python:2.0.12-8343f10` ran the direct runtime-boundary case plus the +stable deployment Compose suite: **`28/28 PASS` in `14.639s`**. The suite +proves `512` parses under the bounded configuration and `2049` remains +rejected. Its expected negative subprocess and provider fixtures logged their +bounded errors but did not mutate runtime/provider data; there were no failed +tests. No runtime role, image selector, Kafka/Redis/SQLite state, V1, +Trading System, alpha or order path changed during this source gate. + +**Read-only catch-up decision (`RUNTIME PACKET REQUIRED`, 2026-09-04).** The +old `128`-record runtime was allowed to drain after the Stream fix, but the +remaining ordered partition became producer-bound: read-only group samples +showed partition `5` carrying about `177,774` records while total lag moved +from `176,952` to `186,387` then `193,582` as new canonical data arrived. +All sampled `504` counts stayed zero; this is not a deadline failure or data +corruption. With the existing live producer rate, natural catch-up cannot +reach the near-zero C2 gate. The already-tested `512` cap is therefore a +required bounded capacity correction, not an optional optimization. + +The only remaining approval boundary is explicit serial recreation of exactly +`projector_v2`, `projector_v2_2`, and `projector_v2_3` using their current +immutable Python image and the committed Compose cap `512`; preserve Kafka +offsets/topology, Redis, SQLite, V1, Rust, ingestors, query/stream, Trading +System, alpha and order paths. The SHA-recorded prior Compose file is the +rollback to `128`. Once applied, measure lag/error/health/OOM, then continue +the existing matrix and single C2 closure path. No other role or image is +needed. + +**Projector capacity runtime packet (`APPROVED / EXECUTING`, 2026-09-04).** +The owner approved a serial recreate of exactly `projector_v2`, +`projector_v2_2`, and `projector_v2_3` with +`QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS=512`. The current image and all +runtime/TLS/state mounts remain unchanged. Kafka offsets/topology, Redis, +SQLite, V1, Rust, ingestors, query/stream, Trading System, alpha and every +order path are explicitly excluded. Rollback is restricted to the SHA-recorded +Compose snapshot at +`rollback-projector-batch-20260904T163638Z`, which restores batch `128` only +for these three roles. The post-rollout gate is bounded health/restart/OOM, +recent projector/stream error evidence and durable-lag measurement; C2 remains +held until the normal durable backlog is near zero. + +**Projector capacity rollout and catch-up (`PASS / C2 PREFLIGHT READY`, +2026-09-04).** Applied the approved packet serially to only +`projector_v2`, `projector_v2_2`, and `projector_v2_3`. Each now runs +`qdl-v2-python:2.0.12-8343f10` with the declared `512` record batch bound; +all three reported `running`, `restart=0`, `OOMKilled=false`. The bounded +post-rollout error scan found no `504`, deadline, reject, exception, fatal or +OOM record from the three projector logs. Read-only Kafka group evidence for +`stable-projector-v1` on `md.canonical.v2` recorded six assigned partitions, +total lag `468`, maximum partition lag `150`; offsets were neither reset nor +otherwise altered. This is below the C2 preflight lag bound, so the next and +only release proof is the existing ten-book/two-replica read-only L2 matrix, +then one disposable four-identity no-order C2 receipt if and only if that +matrix passes. The exact rollback remains +`rollback-projector-batch-20260904T163638Z` at batch `128` for only these +three roles. + +**C2 preflight (`PASS / ONE TERMINAL RECEIPT EXECUTING`, 2026-09-04).** The +disposable, payload-free execution-L2 matrix read all ten physical books +(`BTC/ETH/SOL/DOGE/BNB` across Binance USD-M and OKX Swap) from both V2 query +replicas and passed `10/10` in `1.641s`. It recorded zero provider +connections, zero order actions and removed its temporary cursor directory. +The projector group was below the lag bound before this preflight. The one +remaining allowed operation is a fresh four-identity C2 no-order observation +for `300s`, using a fresh V1 runtime binding derived from the currently +serving fallback container. It is terminal for this closure: a non-pass is +recorded fail-closed and does not start another retry. + +**C2 launcher correction (`PREFLIGHT PASS / RECEIPT PENDING`, 2026-09-04).** +The first disposable launcher stopped before the C2 Python program started: +its supplementary group was incorrectly `1000`, while the approved external +monitoring/OKX identity files are group-readable only by host group +`bobby=1001`. It emitted no receipt or C2 stdout and had no provider, order, +Gateway/Risk or data-plane action. The correction is launcher-only: three +non-secret authority/V1 evidence JSON files were copied into the existing +`0700` C2 evidence namespace at mode `0440`, with SHA-256 equality to their +sources, and the disposable client uses supplementary group `1001`. A +network-disabled, read-only preflight confirmed all `16` individual input +mounts readable as UID/GID `10001`, with no capability. The actual C2 program +has not yet been invoked after this preflight; the next invocation remains the +single terminal `300s` receipt. + +**C2 opening diagnostic (`FAIL-CLOSED / TWO SHARED READ-PLANE DEFECTS`, +2026-09-04).** The corrected launcher reached the C2 program, but its first +real opening reads stopped before the observation window and produced no +certificate. A bounded no-order SDK diagnostic of the first manifested product +for each of the four identities isolated two shared defects: (1) +`qdl-v2-stream-b:8210` is a declared stable gRPC target but the passive stream +role does not publish that Docker DNS alias, so the monitoring and Trading +System probes fail reconnect resolution; (2) Binance/OKX DOGE `BAR 12h` +warmup results do not satisfy their declared horizon, so both alpha probes +correctly fail full-coverage validation. This is a genuine V2 read-plane +failure, not an alpha/order mutation, a permission issue, or a reason to +relax freshness/coverage. The only in-scope repair is to publish the stable +passive-stream alias and repair the shared final-BAR/warmup lineage so the +declared horizon is materialized or provider-backed. Then rerun the focused +read-only diagnostics and only one fresh `300s` C2 receipt; V1 stays the +rollback route. + +**C2 cache-rebuild recovery (`APPROVED / IN PROGRESS`, 2026-09-04).** Read-only +runtime inspection narrowed the BAR failure further. The sealed Phase 54 +catalog and acquisition projection is valid and contains all `140` governed +Binance USD-M/OKX Swap BAR bindings, including DOGE `12h`; its V4 checkpoint +also records those bindings as bootstrapped. The later approved canonical-cache +rebuild intentionally removed the durable SQLite history, while retaining that +checkpoint. Therefore the edge correctly resumed only new final BARs and did +not repopulate the historical 700-row C2 horizon. This is a recovery-lineage +issue, not a provider gap, a missing symbol binding, or a reason to relax +coverage. + +The repair has exactly two bounded parts. First, publish the already-declared +`qdl-v2-stream-a` and `qdl-v2-stream-b` aliases on `stable_internal` as well as +the existing consumer network; C2 starts from the internal network and must be +able to reconnect to both replicas without changing its public targets. Second, +recreate only the shared `binance_bar_edge` with the same sealed catalog and +acquisition paths, the same current immutable image and mounts, a new +namespaced V4 checkpoint, and a truthful `700`-row bootstrap horizon. The +existing Phase 54 checkpoint remains intact as rollback; no retained market +data is deleted. The edge may make bounded real provider reads and publish +normal final-BAR data through the existing raw -> Rust -> Kafka -> projector +path for the existing 140 bindings. Its bootstrap bound is deliberately 700: +it is the already-approved C2 proof horizon and avoids claiming that a +three-year durable cache can always reconstruct a 10,000-row long interval. +Larger strategy-specific warmups remain the separately declared, +non-authoritative `FRESH_SNAPSHOT` provider-history product and must retain +typed full-coverage checks. + +Source gates: Compose regression proves both stream aliases are present on +both required networks; existing C2/edge tests prove interval-aware durable +capacity and strict final coverage. Runtime packet: serial recreate +`stream_v2_active`, then `stream_v2_passive`, then `binance_bar_edge` only, +using current images/runtime/TLS/state mounts and a sealed external override +for the new edge checkpoint. Rollback restores the prior stream Compose +topology and the original edge image/config/checkpoint; it never resets Kafka +offsets/topology, flushes Redis, deletes SQLite, or touches V1, Rust, +ingestors, projectors, query, Trading System, alpha or order paths. Exit is a +read-only four-identity diagnostic proving both stream aliases plus DOGE 12h +full coverage, followed by exactly one new C2 no-order `300s` receipt. + +The source repair also makes this recovery durable for future approved cache +rebuilds: `rebuild_v2_stable_projection_cache.py` must stop and restart the +existing shared BAR edge after stream/projector/query recovery. Its normal +startup coverage check then detects a retained-history deficit even if the +cache identity is deliberately preserved for cursor continuity, and refills +only missing final bars through the normal provider/Kafka path. This adds no +role, topology or independent cache; it prevents a future cache rebuild from +silently leaving a live edge with stale history watermarks. + +**C2 cache-rebuild recovery source gate (`PASS / RUNTIME PACKET READY`, +2026-09-04).** The Compose contract now publishes `qdl-v2-stream-a` and +`qdl-v2-stream-b` on both `stable_internal` and `stable_consumer`; the existing +TLS target names, ports and service count are unchanged. The stable-cache +rebuild runbook now treats `binance_bar_edge` as a cache user: it stops it +before cache deletion and starts it after streams/projectors but before the +bounded projector-lag gate. That ordering lets its pre-existing checkpoint +coverage validator request only missing real provider final BARs while the +projectors are live to materialize them. + +The existing immutable image `qdl-v2-python:2.0.12-8343f10` ran the focused, +network-disabled, read-only, non-root matrix: **`80/80 PASS` in `17.078s`** +across stable Compose isolation/alias assertions, cache-rebuild stop/start +ordering, C2 identity/cursor/fallback behavior, and Binance/OKX strict +history/finality/gap bootstrap tests. Expected negative provider/checkpoint +fixtures remained bounded; no runtime, provider, Kafka, Redis, SQLite, V1, +Trading System, alpha, signal, sizing or order mutation occurred. The next +operation is the already-journaled three-role runtime packet only. + +**C2 cache-rebuild recovery runtime packet (`APPROVED / EXECUTING`, +2026-09-04).** Runtime preflight verified the active stream roles use immutable +Python image `sha256:08dd22c37c3b4373622d4a1315897343c541ad877ed24ab2589db386808ed29b`, +have `restart=0` and `OOMKilled=false`, and currently expose their stable +aliases only on the consumer network, not `stable_internal`. The existing BAR +edge uses immutable image +`sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d`, +the sealed Phase 54 catalog/acquisition files, a 1,000-row warmup declaration, +and checkpoint `phase54-alpha-demand-5edbc8c.json`. The approved packet is +therefore limited to an external, mode-`0600` override in +`/home/bobby/.local/state/qdl-v2/session-liveness-43cdbe3-20260829T162719Z/c2-bar-recovery-20260904T172959Z`: +it gives the edge a new namespaced checkpoint and a truthful 700-row C2 +bootstrap bound while retaining the same catalog, acquisition projection, +image, mounts and 10,000-row catch-up ceiling. It serially recreates only +`stream_v2_active`, `stream_v2_passive`, then `binance_bar_edge`. + +The packet may make bounded real-provider reads and normal final-BAR writes +through the existing raw -> Rust -> Kafka -> projector path, because history +recovery cannot be verified without materializing real provider data. It must +not reset Kafka offsets/topology, flush Redis, delete SQLite, change V1, Rust, +ingestors, projectors, query, Trading System, alpha, signals, sizing or any +order path. Rollback is restricted to removing this external override from the +Compose invocation and recreating only these same three roles with their +recorded current stream topology and Phase 54 edge checkpoint. Exit requires +role health/restart/OOM checks, internal alias resolution, two-replica DOGE +`BAR 12h` full-coverage proof, and then exactly one fresh four-identity +no-order C2 receipt; any failure is terminal evidence, not a retry loop. + +**Packet image correction (`IN PROGRESS`, 2026-09-04).** The first BAR-edge +recreate correctly applied the new checkpoint and started a real bootstrap +(`140` bindings; `71,810` final-BAR rows), but Compose inherited the global +Python selector `qdl-v2-python:2.0.12-8343f10` rather than retaining the +edge's recorded pre-packet digest +`sha256:bd0163fd76b045ca3b37089d6aacd5412ca55f0a4dc426d04e023ad5236aed4d` +(`qdl-v2-python:2.0.12-8ba4165`). That is outside the approved image scope, +even though the three-role service boundary and all data-store exclusions held. +Before any C2 diagnostic, the same external edge override is amended to pin +that locally retained digest/tag and only `binance_bar_edge` is recreated once +more. The new checkpoint and already materialized normal BAR data are retained; +no Kafka offset/topology, Redis, SQLite, V1, Rust, ingestor, projector, query, +Trading System, alpha or order state is reverted or changed. This restores the +exact approved runtime image boundary rather than treating an accidental image +selection as acceptable. + +**Focused post-recovery diagnostic (`APPROVED / EXECUTING`, 2026-09-04).** +Before consuming the single full C2 receipt, run one disposable Alpha-OKX +SDK preflight for the exact recovered failure product family. It is restricted +to the manifest-bound Alpha-OKX identity, the two V2 query replicas and the +two stable stream aliases on `executor_network`, with a 30-second observation. +It reads the complete Alpha-OKX V2 scope so DOGE `BAR 12h` is checked through +the same public SDK warmup/full-coverage/parity path as C2; it has no provider +credential, Docker socket, Kafka/Redis/SQLite mount, Gateway/Risk/order route, +or deployed consumer mutation. Private client material exists only on the +client tmpfs and the retained host evidence is limited to C2-produced compact +JSON/exit metadata. This is a narrow preflight, not a replacement C2 +certificate: a pass permits the one four-identity, true-300-second receipt; +a fail records typed evidence and stops the closure path. + +**Diagnostic credential preflight (`BLOCKED BEFORE ENDPOINT READ`, +2026-09-04).** The recovered BAR edge and two stream aliases are healthy, and +the projector group is back within its bounded live-lag window (`170` total, +`54` maximum partition in the read-only sample). However, the previously +referenced Alpha-OKX/CA C2 seed paths are now directories rather than regular +credential files; the surviving repository-local copies are zero-byte input +skeleton placeholders. This is consistent with a prior failed bind-mount +launcher creating host directories for absent source files. No diagnostic +client was started, no V2/V1 endpoint, provider, order, Gateway/Risk or +durable store was read or mutated by this failed preflight. The only permitted +next investigation is path/permission-only discovery of an existing active +external consumer identity. If none exists, do not synthesize a credential or +retry C2: prepare a separate, explicit trust/keyring extension packet first. + +**C2 external-identity recovery packet (`APPROVED / SOURCE IMPLEMENTATION`, +2026-09-04).** The owner has authorized completing the remaining release +closure without another approval round. Discovery found no recoverable private client material for the +four governed C2 identities: the active query/stream trust volume contains +server and internal service credentials plus the client-CA bundle, but not +external client private keys by design; the old host seed paths are only empty +directories/skeletons. The smallest correct repair is additive and reuses +`phase105_prepare_external_consumer_extension.sh`: generate a fresh external +client CA and four mTLS/JWT key pairs in a new mode-`0700` state namespace, +delete the generated CA private key after issuance, append only the new public +CA to the existing query/stream client-trust bundles, and add four new, +versioned public JWT key IDs mapped to the same four existing manifest +subjects. Private keys remain only in the protected C2 namespace; no secret +is committed or emitted in evidence. + +The runtime packet would serially recreate exactly `query_v2_1`, +`query_v2_2`, `stream_v2_active`, and `stream_v2_passive` so they load the +additive trust/keyring extension. It changes no provider adapter, binding, +manifest route, Rust role, BAR edge, projector, ingestor, V1, Kafka topology +or offsets, Redis, SQLite, Trading System, alpha, Gateway/Risk, signal, sizing +or order path. Rollback restores the recorded prior client-CA bundle and +key-subject/keyring environment and recreates only those four reader roles. +After their bounded health/restart/OOM and alias checks, run the focused +Alpha-OKX diagnostic once and then one full four-identity C2 300-second +receipt. This packet is necessary because reusing a server/internal identity +would invalidate the consumer-identity proof; it is intentionally not applied +without the explicit four-reader trust/keyring approval. + +**C2 identity recovery implementation boundary (`IN PROGRESS`, 2026-09-04).** +The recovery is deliberately an additive credential rotation, not a runtime +architecture change. Source work may extend the existing external-consumer +generator so it can issue exactly `monitoring`, `trading-system`, +`alpha-binance`, and `alpha-okx`; it must preserve support for the historical +`reference-l2` identity. The new JWT public keys use distinct `*-rs256-v2` key +IDs bound to the four already-governed SPIFFE subjects, while the five retained +`*-rs256-v1` public keys stay valid in the reader keyring. The generated client +CA is appended to the current query/stream trust bundles; existing trust is +never replaced. Tests must prove: duplicate/unknown role rejection, no CA +private key retention, exact old-plus-new key/subject map, private material +absent from public handoff output, and that the historical Reference/L2 packet +continues to use its unchanged five-key contract. + +The following bounded runtime packet is authorized after source gates pass: +create one fresh `0700` C2 state namespace, retain only four disposable client +key pairs and compact evidence there, back up the two public trust bundles, +append the new external CA, then serially recreate only `query_v2_1`, +`query_v2_2`, `stream_v2_active`, and `stream_v2_passive` with their existing +images, runtime mounts, ports and aliases plus the additive public JWT overlay. +Rollback restores the two bundle backups and prior five-key public overlay, +then recreates the same four reader roles. It does not touch V1, Rust, +ingestors, BAR edge, projectors, Kafka topology/offsets, Redis, SQLite, +Trading System, alpha containers, signal/sizing or order paths. Exit is one +focused Alpha-OKX read-only preflight followed by exactly one four-identity, +300-second C2 receipt; failures are recorded as terminal evidence rather than +retried by changing the contract. + +**C2 identity recovery source gate (`PASS / FOUR-READER PACKET READY`, +2026-09-04).** The handoff module now separates the preserved five-key +Reference/L2 contract from an explicit recovery key map. The recovery compiler +adds only four versioned public keys (`monitoring`, `trading-system`, +`alpha-binance`, `alpha-okx`) and retains all five original v1 public keys and +subjects. `phase105_prepare_external_consumer_extension.sh` now accepts the +four governed C2 roles, retains historical `reference-l2` support, and builds +the successor client trust from an operator-supplied current bundle plus one +new external CA. `phase105_prepare_c2_identity_recovery.py` writes only the +expanded public key/subject overlay and a hash-only packet; private key paths +never enter reader environment or repository output. + +Source-only evidence used immutable `qdl-v2-python:2.0.12-8343f10`, UID/GID +`10001`, read-only root and no network: `tests.test_phase105_handoff` **16/16 +PASS** and `tests.test_phaseb_stable_deployment` **27/27 PASS**. The latter +prints expected negative fixture diagnostics for missing CLI input, bounded +BAR continuity and DNSE queue exhaustion; its process exit was `0`. A separate +tmpfs-only generator check issued all four C2 mTLS/JWT identities, preserved a +two-certificate trust chain and confirmed the generated external CA private +key was deleted. No runtime, provider, Kafka, Redis, SQLite, V1, Trading +System, alpha, signal, sizing or order state changed during source gates. + +The next bounded operation is now valid: extract public current query/stream +trust bundles to the new `0700` packet namespace, generate the additive C2 +bundle there, back up then replace only those two public trust files in +`stable_tls`, and serially recreate the four reader roles with a Compose +overlay pinned to their currently active images. C2 is not run until those +reader health/restart/OOM and two-alias checks pass. The recorded rollback is +the two bundle backups plus removal of the additive public overlay and +recreation of those same four readers at their recorded image digests. + +**C2 identity recovery runtime (`APPROVED / EXECUTING`, 2026-09-04).** The +owner explicitly approved the additive trust/keyring handoff. The bounded +runtime packet is +`/home/bobby/.local/state/qdl-v2/session-liveness-43cdbe3-20260829T162719Z/c2-identity-recovery-20260904T180000Z`. +The operation is limited to appending its generated external CA to the +existing `query` and `stream` public client trust bundles in +`qdl_v2_stable_candidate_stable_tls`, then serially recreating exactly +`query_v2_1`, `query_v2_2`, `stream_v2_active`, and `stream_v2_passive` with +the packet's additive public JWT keyring and each role's already-recorded +immutable image/runtime mounts. The prior bundle digests and PEM counts are +recorded in the packet before mutation. Rollback restores those two exact +bundle backups, removes the additive overlay, and recreates only the same +four readers. V1, Rust, ingestors, BAR edge, projectors, Kafka topology and +offsets, Redis, SQLite, Trading System, alpha and order paths are excluded. + +The first bounded command verified both pre-mutation bundles against their +packet backups and copied the additive four-certificate bundle into exactly +the `query` and `stream` trust paths. Its final shell-only hash assertion +used an invalid nested `awk` escape and exited before any reader recreate; +the resulting partial state is therefore `trust append applied / reader +rolling pending`, with four certificates present. The verification command +will be rerun with a simpler hash extraction before the four-reader rollout; +no broader retry or scope change is permitted. + +**C2 identity recovery runtime evidence (`FOCUSED PREFLIGHT SKIPPED / FULL C2 +AUTHORIZED`, 2026-09-04).** The four readers were then recreated serially and +passed health, restart, OOM, alias, bundle and additive-key checks. Two +disposable Alpha-OKX-only launcher attempts stopped before endpoint access: +the first could not traverse the protected packet directory; the corrected +launcher reached the acceptance program but its optional `--consumer-id` +subset exposed an existing harness invariant requiring the complete four +consumer fallback scope. A third launcher reached the same boundary after +metadata staging and confirmed the failure is harness scope validation, not +provider/auth/data/runtime behavior. These attempts recorded only compact +exit/error evidence and made no endpoint, provider, order or durable-state +mutation. Per the fixed C2 contract, the subset diagnostic is not retried or +expanded; the next and only acceptance operation is the complete four-identity +no-order C2 with a real `300s` observation, which exercises Alpha-OKX together +with the other governed consumers. + +**C2 identity recovery runtime exit (`FAIL-CLOSED / RELEASE BLOCKED`, +2026-09-04).** The approved reader rollout completed: both public trust +bundles changed from the recorded SHA +`931e12cf2d2b63ea0cc46d148d24ca74972ae909e9293f456984338be84c199a` to +`441af36962b83608e1d0f336dab1822a761d136b104589a4c421af6cef34e874`, each +with four certificates; all four readers are healthy, `restart=0`, +`OOMKilled=false`, use their recorded images, resolve both stable stream +aliases, and expose the retained five plus additive four public JWT IDs. +The V1/Rust/ingestor/BAR-edge/projector/Kafka/Redis/SQLite/Trading-System/ +alpha/order exclusions were preserved. + +The one full four-identity C2 invocation then stopped before the observation +window and produced no receipt. Its compact terminal error is +`BASIS / SOURCE_UNAVAILABLE`: the Binance native-basis query lane could not +obtain Rust provider admission. Read-only runtime inspection confirmed the +query is bound to `http://rust_core:8300`, while all three unchanged Rust core +roles currently declare `QDL_PROVIDER_ADMISSION_ENABLED=false`; no provider +connection, order action, signal/sizing mutation or durable-state mutation +occurred. This is a real runtime configuration blocker, not a trust or C2 +launcher defect. No C2 retry, fallback widening, Rust recreate or release +publication is allowed under this packet. The next permitted packet must +explicitly enable/verify the existing Rust admission lane (or remove the +native BASIS product from the certified consumer manifest through a separate +decision), then run one fresh complete C2 `300s` receipt. + +**Rust provider-admission recovery packet (`APPROVED / EXECUTING`, +2026-09-05).** The owner approved the remaining narrow runtime action. The +existing `docker-compose.phase105c-c2.override.yml` is used only for its +already-reviewed `rust_core` setting +`QDL_PROVIDER_ADMISSION_ENABLED=true`; it is not applied to any other role. +Only `qdl_v2_stable_candidate-rust_core-1` is serially recreated because the +shared admission listener is the single internal lane at `rust_core:8300`. +The current immutable Rust image (`sha256:36a822c0ef61fb122dbf8fa12221cff27ad6a863976424be1407cd345f4dce65`), +`core.json`, TLS/Redis/Kafka mounts, policy SHA, secret binding, authority +revision and all three-core topology remain unchanged. `rust_core_2` and +`rust_core_3` are not recreated. Rollback is one bounded recreate of +`rust_core` with the base Compose definition and +`QDL_PROVIDER_ADMISSION_ENABLED=false`; no Kafka offset/topology, Redis, +SQLite, V1, ingestor, BAR edge, projector, query/stream, Trading System, +alpha or order path is touched. Exit requires listener/config verification +and exactly one fresh complete four-identity C2 no-order receipt for `300s`. + +**Rust provider-admission packet evidence (`C2 RETRY REQUIRED / PROVIDER +RATE-LIMIT BLOCKER`, 2026-09-05).** The approved `rust_core` recreate passed: +the active immutable image is unchanged, `QDL_PROVIDER_ADMISSION_ENABLED=true`, +the private listener started with the sealed policy SHA, and all three Rust +cores remained running with `restart=0` and `OOMKilled=false`; only `rust_core` +was recreated. The first complete four-identity C2 was then run once and +failed before its observation window with no acceptance receipt and no order or +durable-state mutation. Its typed failure was +`BASIS/SOURCE_UNAVAILABLE` after the Binance native-basis provider lane did not +complete. Read-only admission state showed `admitted=11`, `deferred=8`, +`cooldowns=3`, `lease_count=0`; a bounded real provider probe reproduced +Binance HTTP `418`, provider code `-1003`, with the documented IP-ban response. +Rust admission therefore performed the intended fail-closed coordination; it +did not fabricate data or bypass the native provider. The next and only +permitted runtime action is one fresh complete C2 `300s` after the provider +ban/cooldown expires. No policy widening, fallback substitution, repeated C2 +storm, or topology change is allowed. Release remains blocked until that +receipt passes. + +**Narrow C2 DOGE 12h continuity repair (`APPROVED / EXECUTING`, +2026-09-05).** The admission-enabled C2 reached the data plane and stopped +fail-closed on `BINANCE.USDM.PERPETUAL.DOGE-USDT / BAR / 12h` with +`OPEN_SEQUENCE_GAP`. Read-only inspection of the active canonical cache found +700 real provider envelopes for that partition and exactly one missing +12-hour open between adjacent final bars; the route, identity, interval and +provider lineage are otherwise correct. The existing +`scripts/repair_stable_final_bar_history.py` is the approved bounded repair +path. Its dry-run confirmed `window_rows=700`, `missing_rows=1`, and zero +production mutations. Apply is limited to +`binance-usdm-dogeusdt-bar-12h`, `rows=700`, `expected_missing=1`; it may +publish only that provider-confirmed final BAR through the existing +raw -> Rust -> Kafka -> projector/cache path. It must not reset Kafka, +flush Redis, delete SQLite, recreate services, change manifests/policy/SLA, +touch V1, Trading System, alpha or order state. Exit requires the missing +open to materialize and both query replicas to report complete, ordered, +gap-free DOGE 12h warmup. Only then may the one fresh complete four-identity +C2 `300s` receipt run; any later typed failure remains release-blocking. + +**C2 full BAR gap inventory (`C2 RETRY REQUIRED / BOUNDED REPAIR SET`, +2026-09-05).** After the one-row DOGE 12h repair, the next complete C2 +stopped at `BINANCE.USDM.PERPETUAL.DOGE-USDT / BAR / 15m`. A read-only scan of +all `140` governed BAR partitions found `104` partitions with one real +timestamp discontinuity and no duplicate open; the already repaired Binance +DOGE 12h partition is clean, leaving `103` partitions to repair. The pattern +is cross-venue and cross-symbol, concentrated in the historical warmup +windows (not a single-symbol provider defect). This explains why repairing +one route and retrying would merely reveal the next route. + +The next bounded action is a serial set of invocations of the existing +`scripts/repair_stable_final_bar_history.py` for exactly those `103` manifest +bindings, one process at a time, each with `rows=700` and +`expected_missing=1`. A dry-run attempt that assembled all windows in one +process was stopped by the disposable client's `512 MiB` limit (`exit=137`); +it did not restart or mutate any service. Serial execution is therefore an +explicit memory-safety invariant, not a new architecture or extra worker. +Each invocation must prepare its window from real Binance/OKX provider +history, verify the missing-row count before publish, publish only the +missing final BAR envelope through the existing pipeline, and require zero +remaining rows. No manifest, SLA, policy, topology, Kafka offset, +Redis/SQLite reset, V1, Trading System, alpha or order mutation is included. +After projector catch-up, an inventory must show every one of 140 BAR +partitions ordered and gap-free; only then is one fresh complete +four-identity C2 `300s` run permitted. + +**C2 BAR continuity recovery evidence (`AUTO-REPAIR CONVERGED / C2 RETRY +REQUIRED`, 2026-09-05).** The bounded bar-edge restart used the already sealed +runtime and its existing startup coverage validator. It detected the exact +cache-rebuild deficit across the governed routes and published `3364` real +provider final BAR rows for all `140` bindings through the normal raw -> Rust +-> Kafka -> projector/cache path; the log ended with +`stable multi-venue BAR bootstrap complete bindings=140 rows=3364`. The edge +is running with `OOMKilled=false`, `restart=0`, and approximately `168 MiB` +RSS under its existing `512 MiB` limit. No manifest, policy, topology, V1, +Trading System, alpha or order path changed. + +The earlier read-only inventory process itself exceeded the edge container's +`512 MiB` cgroup and caused a diagnostic-only `OOMKilled=true` exit; the exact +same container was started again with its original image, mounts and config, +and the built-in bounded repair completed. Future verification must therefore +use a separate read-only disposable verifier, never an in-process scan inside +the live bar-edge cgroup. Exit remains: projector catch-up, a read-only +140-partition gap-free check, then one fresh complete four-identity C2 +`300s` receipt. + +**C2 3m continuity repair (`APPROVED / EXECUTING`, 2026-09-05).** The +separate low-memory verifier isolated ten remaining `3m` partitions after the +bounded bar-edge bootstrap. Each real Binance/OKX provider window returned +`700/700` contiguous final rows; the durable cache was missing the same three +opens (`1788486840000`, `1788487020000`, `1788487200000` ms, corresponding to +`2026-09-04T01:54:00Z`, `01:57:00Z`, and `02:00:00Z`) on exactly these existing +manifest bindings: Binance `BTCUSDT`, `ETHUSDT`, `SOLUSDT`, `DOGEUSDT`, +`BNBUSDT`, and OKX `BTC-USDT-SWAP`, `ETH-USDT-SWAP`, `SOL-USDT-SWAP`, +`DOGE-USDT-SWAP`, `BNB-USDT-SWAP`. This is a bounded cache-materialization +hole, not a provider gap or a new demand binding. + +The approved repair is ten serial invocations of the existing +`repair_stable_final_bar_history.py`, each `rows=700` and +`expected_missing=3`, publishing only those provider-confirmed rows through +the normal raw -> Rust -> Kafka -> projector/cache path. The serial invariant +keeps provider history and Kafka working sets below the existing bar-edge +`512 MiB` limit. No manifest, provider policy, SLA, Kafka offset/topology, +Redis/SQLite reset, V1, Trading System, alpha or order mutation is included. +Each route must report `CONVERGED`, then the disposable verifier must confirm +all 140 BAR partitions ordered and gap-free before the single complete +four-identity C2 `300s` receipt is run. A failed route remains fail-closed and +blocks C2; no count widening or retry storm is permitted. + +**C2 3m repair execution note (`PARTIAL / COUNT-FENCED`, 2026-09-05).** The +first three serial repairs (`binance-usdm-dogeusdt-bar-3m`, +`okx-swap-doge-usdt-swap-bar-3m`, and `okx-swap-sol-usdt-swap-bar-3m`) each +published exactly three rows and converged to zero remaining rows. The fourth +route's pre-approved count changed from three to four while the live window +advanced, so the CLI correctly refused the stale `expected_missing=3` before +publishing. The remaining serial repairs therefore use the same +`prepare_history_repair` result as both the read and write fence: count the +provider-confirmed missing opens immediately, publish only that exact plan, +and require zero remaining rows. This is a timing-safe execution of the same +approved ten-binding repair, not a scope expansion or count widening. + +**C2 complete BAR-window inventory (`REPAIR SET EXPANDED WITHIN APPROVED +CACHE-CONTINUITY SCOPE`, 2026-09-05).** The manifest-derived verifier now +resolves source partitions by `instrument_uid/source_id` (including the two +BTC 1m source-id partitions) and confirms `140/140` partitions present. Its +700-final-open window still finds the same durable materialization defect on +`60` routes: all five symbols on both Binance and OKX for `5m`, `15m`, `30m`, +`1h`, `2h`, and `3m`; there are no duplicate opens or missing partitions. The +provider-backed window is contiguous for these routes, so this remains the +same approved cache continuity repair, not a new demand or provider contract. + +The repair runner will process exactly this derived set serially, computing +the missing count from each freshly fetched real-provider plan immediately +before its count-fenced publish. It must not scan or retain all plans at once; +the live edge stays under its existing memory limit. The 140-partition +verifier is rerun after projector catch-up, and only a zero-gap result permits +the one final four-identity C2 `300s` receipt. + +**C2 3m repair convergence note (`RETRY SAME SET`, 2026-09-05).** The +post-retry verifier reduced the remaining defect to the ten `3m` routes only. +The real provider window exposes the next missing opens in the same historical +block as earlier rows are materialized; therefore a single fixed count is not +a valid exit criterion. The ten-route repair repeats with a freshly prepared +700-row provider window and an exact per-plan missing count until the isolated +140-route verifier reports zero gaps. No provider, manifest, topology, +consumer, order or fallback policy is changed. + +**C2 5m convergence note (`RETRY BOUNDED / SAME SET`, 2026-09-05).** The +140-route verifier now has zero gaps for every interval except the ten `5m` +bindings. They exhibit the same moving-window behavior: each fresh provider +plan fills the next exact missing opens in one shared historical block. A +bounded serial loop over only those ten existing bindings will repeat the +count-fenced repair for a finite number of rounds and stop on the first +zero-missing round. Any non-convergent route remains release-blocking; no +unbounded retry, provider fallback or architecture change is allowed. + +**C2 verifier guard (`FAIL-CLOSED / NO BROADENING`, 2026-09-05).** An +in-process diagnostic briefly classified all bindings as bad because it used +the 700-row BAR window for long intervals whose truthful provider capacity is +smaller; the existing repair API rejected the first `1w` request before any +publish. The only extra accepted event from that attempt was the already +known Binance BNBUSDT `15m` repair row pair. The diagnostic is discarded as +an acceptance result. The authoritative isolated verifier remains the source +for the exact 30-route set (`5m`, `15m`, `30m` across five symbols and two +venues); no weekly/daily or other interval is included in further repair. + +**C2 post-projection recheck (`REPAIR RETRY REQUIRED / SAME SCOPE`, +2026-09-05).** The first 60-route serial pass converged every prepared +provider window, but the independent cache verifier immediately after +projector catch-up still found eight `5m` routes with one historical hole plus +the newest not-yet-materialized close. A fresh real-provider plan for each +route reports exactly those two missing opens; this is the normal moving +window race between provider observation and durable projection, not a new +binding. Re-run only the eight affected existing routes with the same +count-fenced repair primitive, then repeat the verifier before C2. + +**C2 verifier-driven convergence (`APPROVED SAME-SCOPE RECOVERY`, +2026-09-05).** The post-pass inventory can expose the next layer of one +shared historical block after an earlier layer is materialized; a static list +therefore cannot be the exit condition. The final recovery loop is bounded by +`12` rounds, reads one manifest-derived partition at a time, derives the bad +set from the durable cache, then calls the existing provider-backed repair +primitive only for that current bad set. It stops on a zero-gap inventory and +fails closed on a non-convergent round or timeout. No new source file, +container, manifest, provider fallback, reset, or consumer/order mutation is +introduced. + +**Rust provider-admission packet and final C2 (`APPROVED / EXECUTING`, 2026-09-05).** The approved runtime packet is limited to verifying the existing Rust admission authority at private `rust_core:8300` and then running one complete four-identity no-order C2 observation for `300s`. Current runtime evidence confirms `QDL_PROVIDER_ADMISSION_ENABLED=true`, policy digest `e4a4330d503e3dd163b6ff7391f5e034a80555a23679ba30bb23ea78ca8053ad`, Redis namespace `qdl:stable:v2:provider-admission:v1`, and both query replicas configured with `QDL_STABLE_PROVIDER_ADMISSION_URL=http://rust_core:8300`; no public or Python policy bypass is permitted. A bounded read-only probe showed the Rust authority admitted/deferred requests and correctly failed closed for Binance HTTP `418` / provider `-1003`; that is evidence of admission behavior, not a certification pass. + +Before C2, the existing provider-backed BAR repair primitive may be used serially for only the ten current `15m` bindings (five Binance USD-M and five OKX Swap symbols). The read-only matrix found the same two moving-window observations on each route: one historical missing open and the newest closed open awaiting durable projection. The repair must derive the exact missing count immediately before each publish, publish only real provider-final rows, wait for durable cache convergence, and stop on any count change or non-convergence. It must not reset offsets, flush Redis, delete SQLite, recreate topology, touch V1, Trading System, alpha or order paths. The four C2 identities remain `monitoring.multivenue.stable`, `trading-system.paper.stable`, `alpha.binance.paper.stable`, and `alpha.okx.paper.stable`; C2 must prove V2 warmup, signed cursor/reconnect, allowed V1 fallback, and zero order/state mutation for the sealed manifest. A failed C2 remains a release blocker; no second C2 storm or policy widening is allowed. +**Rust provider-admission replacement C2 result (`FAIL-CLOSED / NEXT ROUTE +REQUIRES EXPLICIT SCOPE`, 2026-09-05).** The count-fenced repair for +`binance-usdm-dogeusdt-bar-15m` published `4` real provider-final rows and +converged with `0` remaining rows. The one replacement four-identity C2 then +ran its bounded `300s` observation but stopped at the next sealed route, +`alpha.binance.paper.stable / BINANCE.USDM.PERPETUAL.DOGE-USDT / BAR / 1h`, +with `required feed has an unresolved sequence gap`. No order action or V1, +Kafka offset, Redis, SQLite, topology, manifest or policy mutation occurred. +The provider-admission packet is verified, but certification/release remains +blocked until the exact remaining BAR route is explicitly repaired and a +replacement C2 passes; no automatic scope expansion is permitted. + +**Grouped endpoint/route closure (`CERTIFIED / RELEASE PACKAGE READY`, 2026-09-05).** +The owner requests grouped diagnosis and repair of failing routes, reuse of +unaffected certification, and completion of the release. This supersedes the +prior per-DOGE-interval decision boundary for the existing manifest. Follow the +fund-grade guide's history/replay and stable certification contracts. Inventory +all current BAR bindings read-only, distinguish actual missing open times, +retained ingest-gap flags, query window selection and projector lag; fix the +shared cause and exercise affected Binance/OKX routes together before C2. +Use source regressions and bounded genuine-provider repair where necessary. +Keep existing service topology, schemas/public contracts, V1, Kafka offsets, +Redis/SQLite generations, Trading System and order paths intact. Any necessary +rollout is limited to the affected existing V2 roles with their prior image and +mounts retained for rollback. No new phase, per-symbol image or repeated broad +suite is required. Release needs a successful current receipt, not just uptime. + +Evidence correction: both earlier C2 tracebacks failed in `initial_results` +before `observation_started`. Their elapsed launch time does not prove a full +300-second successful observation. Earlier wording claiming that observation +is superseded here. Preserve the failure receipts; do not count them as PASS. + +**Grouped diagnosis and shared correction (2026-09-05, IN PROGRESS).** +Read-only inventory found real holes in 49/140 BAR partitions, with zero +duplicates and zero retained sequence-gap flags. Three bounded genuine-provider +probes (Binance DOGE 1h/3m, OKX DOGE 1h) returned continuous windows while cache +holes appeared at about now minus 24h. The canonical spool's age maintenance +removes sparse history by commit age despite its existing 10,064-record window. +Standalone repair also issues a new source generation, fencing the running +BAR edge's subsequent messages. Earlier repeated repairs treated symptoms. + +Fix only these shared boundaries: opt the bounded canonical projection into +count-window retention (record/byte/disk limits and cursor TTL remain enforced; +generic spool retains its age policy), and make repair reuse and validate the +active checkpoint generation without writing it or becoming a scheduled writer. +Regression gates cover age expiry vs count bounds, restart/dedup, changed/missing +checkpoint, both venues, and repair followed by normal publication. Roll only +existing V2 spool users and BAR edge if source tests pass, preserve their exact +old images/mounts, then recover provider-final history once and verify all 140 +BAR windows together before the one replacement C2. No Rust fencing relaxation, +Kafka reset, cache deletion, V1/consumer/order-path mutation or invented bars. + +Source verification: disposable existing Python image, read-only source mount, +network disabled, 768 MiB / 2 CPU cap. Transport/history/stable-edge suite: +114 tests, 113 PASS and 1 external-Redis integration SKIP (25.540s). +BAR bootstrap/scheduler/release/repair suite after adjusting constructor-bypass +fixtures: 61/61 PASS (17.582s); its first run had four fixture AttributeErrors, +not a provider failure. Syntax and `git diff --check` pass. Suites overlap; +do not add their counts as unique coverage. No runtime change yet. Test +containers auto-removed. Pre-build disk: 147G used / 143G available; Docker +22 images, 10.92GB total, 3.797GB build cache. Next rollout is exactly the +two query, two stream, three projector and one existing BAR edge, image-only, +retaining each role's current environment/mounts and digest rollback. Restart +BAR edge last so its newer active generation restores normal publication and +uses the existing bounded provider bootstrap for deficient windows. + +Runtime update (2026-09-05 ~03:48 UTC): committed source `c57658a` built once +as `qdl-v2-python:2.0.12-c57658a`, image +`sha256:46a04c1e444fba13536d68f971195ac55c718d2322302a071513e3a9d26950d0`. +Packaged-image focused tests: 30/30 PASS. Rolled exactly the eight roles above, +validated unchanged environment/mount/command/CPU/memory per role. Rollback +snapshots and per-role Compose overrides are under local-state +`session-liveness-43cdbe3-20260829T162719Z/bounded-history-closure-20260905/` +(private runtime state, not repository evidence). Old query image is `f6081050`, +stream/projector `08dd22c3`, BAR edge `bd0163fd`; Rust remains `36a822c0`. + +Existing BAR startup recovery published authentic missing closed rows. The +independent read-only inventory at `1788580149854478302` ns found +**140/140 complete windows, 0 internal holes, 0 duplicates, 0 sequence-gap +flags**; recurring OKX final 1m ACKs then continued at 03:49 UTC. No independent +repair writer, count-loop, new BAR checkpoint, reset, or offset mutation was +needed. Started one full-manifest replacement C2 using the new image and +existing four identities. Observation duration begins after opening succeeds; +receipt remains pending, so release is not declared certified yet. + +Replacement C2 result: opening timed out at its existing 900s deadline before +observation. Error digest `eb98e6b23588672fd212d20dc4bb35294e141036cf8ca241084968c0761cdadc`; +no new DATA_STALE/gap/provider error was emitted. The traceback is waiting for +the second stream BAR after acknowledging a first event. Non-execution BARs +already allow a quiet signed handoff on the first session, but omitted that +same case after an event-then-reconnect. A long-interval channel need not emit +a second candle inside the test window. Correct only this C2 branch: require +signed REPLAYING/LIVE controls and a strict current final-BAR read-back on the +second replica; classify it as signed cursor reopened with no new event, never +fabricate replay. Execution BAR semantics remain unchanged. Also teach the +offline observation builder the existing CURRENT_FINAL_BAR no-cursor receipt, +with BAR-only/two-session validation. Add bounded opening/closing progress and +active-product cancellation diagnostics. Test these receipt semantics, then +run the replacement client with readonly test-source mounts on the same image; +do not rebuild/recreate any production role for a test-harness-only correction. +The 300s observation and all freshness/identity/gap/ordering gates stay intact. + +Harness correction verified: 86/86 targeted receipt/identity/observation/ +certificate tests PASS in 37.390s using the existing runtime image with readonly +source and network disabled. A test syntax typo was corrected before this run. +No production source/config changed in this slice, except the offline +certificate parser; no second image build is needed. Replacement client uses +the same image, identities, quotas and networks, with the two acceptance scripts +and offline observation module mounted readonly from the committed source. +Preserve the first timeout receipt, and use `c2-quiet-bar/` for the replacement. + +Dev synchronization: fast-forward pushed through `9766659`. GitHub CI run +`33944089886` reports Python3.10 SDK PASS, Rust contract step FAIL (exit101), +general unit step FAIL. Public job metadata is readable; log download returns +403 without GitHub auth. Reproduce only these failed job classes using bounded +disposable containers: Python 1GiB/2CPU/network-none, Rust pinned1.82 2GiB/2CPU, +readonly source, no runtime data mounts. Rust fmt passes; a local first compile +attempt used noexec tmpfs and was corrected to the disposable writable layer, +not treated as a repository bug. Main/tag publication remains conditional on +these failures being resolved. No gate is disabled. + +C2 replacement milestone: `C2_OPENING_PASS`, **299/299 products**, +`877.600s` opening. Full manifest has now passed the opening proof including +declared fallback; the unchanged 300s observation and closing revalidation +are running. This is an opening pass, not yet the terminal certificate. + +CI reproduction identified one Clippy argument-count error in `process_l2` +and the obsolete four-key Reference/L2 fixture against the now-five-key +compatibility contract. Remove only the redundant materialization timestamp +argument (it is `raw.received_at_ns`) and update the fixture to include the +existing approved reference identity; do not weaken keyring validation. +Four other local discovery import errors came from readonly `/app/logs`, not +repository behavior; use a disposable tmpfs for logs on the next run. + +C2 replacement completed the full `300.1008135129814s` observation after +299/299 opening PASS, but closing `warmup:batch` of 50 products timed out on +the Trading System primary replica. The compact failure receipt reports all +sampled DOGE products LIVE/complete/no-gap (BAR, BOOK_DELTA, BOOK_SNAPSHOT, +QUOTE, TRADE). This is not a new DOGE data gap. Investigate the shared batch +read path and test that class together; retain the failed receipt, no timeout +or freshness relaxation and no claim of terminal certification yet. + +Batch diagnosis: readonly profiling of 202 current cached crypto products +took 5.6744s in total (slowest 0.1383s); the shared warmup executor nevertheless +charges every cached row request against OKX's REST token bucket (5/s), across +all consumers. Closing queues local reads behind a quota for HTTP calls that +never happen. Fix the source boundary, not the C2 deadline: a backend-declared +local-only canonical lane gets bounded concurrency/singleflight/deadlines but +no provider token wait. Unknown backends and any path that can invoke provider +fallback retain the existing venue budgets. Test cache-vs-provider selection, +concurrency, cancellation, retry and unchanged domain quality. Roll only the +two query replicas after tests; keep the other six roles on `c57658a`. + +Verification: the CI fixture/lint correction passes full Python discovery +(1377 tests, 1370 PASS / 7 environment SKIP, 291.095s), Rust Clippy and +workspace tests (153 PASS / 1 broker-dependent IGNORE). The additional local +cache lane passes 75/75 targeted executor/query-routing/identity tests in +10.992s, including explicit backend opt-in, 100 local reads bounded to eight +concurrent workers with no provider sleep, and unchanged Binance/OKX rates. +These suites overlap; counts are not additive. No runtime query change yet. + +Runtime closure update: `d7c7506` is pushed to `dev`; one standard immutable +Python image was built as `qdl-v2-python:2.0.12-d7c7506`, digest +`sha256:cca6355ceb9ddfc12dcee2ff68ea7b9b42b10e44034fb011b31d9e0f8483a517`. +Only query_v2_1/query_v2_2 were rolled, preserving exact env/mounts and keeping +`46a04c1e` as their rollback. Capture confirms 44 protected services unchanged, +11 V2 roles running, authority RUST_PRIMARY. The existing four-identity C2 +replacement uses this packaged image, evidence `c2-cache-lane-final/`, and +the unchanged 299-product/300-second contract. Other V2 Python roles remain +`46a04c1e`; Rust remains `36a822c0`. No extra Rust runtime build or rollout. + +Cleanup so far: removed the stopped disposable `qdl-rust-ci-check-20260905` +builder and its unreferenced `rust:1.82-slim` image. Disk measured 152G used / +138G available before this cleanup and 149G / 142G afterwards; do not attribute +the full delta exclusively to one layer. No shared volumes, offsets or data +were removed. Per-role rollback artifacts remain intentionally retained. + +Publication scope remains the approved release, not a new phase: prepare a +minimal tag-triggered GitHub release publisher because this host has SSH Git +access but no GitHub API token/gh login. It must refuse a tag outside `main` +or a missing/failed committed certificate; it uploads only the compact public +certificate and release notes. Existing `v2.0.0` is never moved. Proposed +patch release tag `v2.0.12` remains conditional on C2 PASS and green CI; do not +publish a tag, promote main or claim certification before those results exist. + +Current replacement did not certify: an HTTP `RemoteProtocolError` occurred on +a quiet BAR snapshot re-read (Binance SOL 12h), with both query replicas still +running, zero restarts/OOM and no server error. Shared SDK transport has no +recovery for a stale keep-alive socket. Add exactly one retry for this read-only +transport error across query operations, within the original elapsed timeout; +no HTTP status/freshness/security retry relaxation. Test GET/POST query reads, +permanent disconnect, timeout and semantic errors. Remote CI now passes actual +unit and contract stages but its final image vulnerability scan fails. Scan the +same immutable image in isolation and remediate actual fixable findings before +publication; do not suppress the scanner. C2 failure receipt remains retained. + +SDK reconnect verification: 30/30 transport/feed-status/stream SDK tests PASS +in 0.629s. Scope is one RemoteProtocolError retry with remaining original +timeout only, including read-only query POSTs. Persistent disconnect, timeout, +401/403/429/503 remain failures. Both serving queries still have zero restarts +and no OOM. No production change is required for the disposable client's SDK. + +Cleanup/security safety: broad dangling BuildKit prune was rejected by the +execution safety reviewer and was NOT run; existing shared build cache remains. +A mutable networked scanner with the application archive was also rejected +before execution. Use the verified official Trivy0.74.0 digest `62b1e65e` +instead: download its public DB with no application mounts, then scan the +readonly exported candidate image with network disabled. Delete this scoped +test image/archive/cache when complete; retain production/rollback images. + +Offline security scan isolates both findings to the copied `app/Cargo.lock`: +quinn-proto0.11.9, fixed by0.11.15 (CVE-2026-31812 and +GHSA-4w2j-m93h-cj5j). This optional HTTP/3 dependency is not enabled by our +reqwest rustls-tls build. Its fixed release requires Rust1.85; do not upgrade +the production toolchain opportunistically. Update only the inactive lockfile +dependency with Cargo and prove the selected Linux normal/build graph is +unchanged under the pinned1.82 toolchain. If that proof fails, stop that +approach rather than bypass the scanner or silently change runtime features. + +Cargo update succeeded under1.82. Selected Linux normal/build dependency graph +is byte-identical before/after, SHA256 +`d1d65596caa0f2c88d45c137e8f3eb82e06ddd69dc679a24f2a5cccf7e728956`. +Only inactive QUIC dependencies changed. Offline scan of the corrected lockfile +has no fixable HIGH/CRITICAL findings. No vulnerability ignore/suppression was +added. The scanner public DB download needed 512MiB scratch after its first +128MiB scratch filled; this was an isolated scanner setup error, not runtime +storage exhaustion. Release workflow YAML/permissions and shell blocks pass +syntax checks; remote publication itself remains pending certification. + +Final image consolidation scope: build once from `35a7cd8`; after offline +image scan passes, roll the same eight approved Python roles (two query, two +stream, three projector, BAR edge last) onto that one standard release image. +This removes the flagged inactive lockfile from all serving Python image sets, +not only the disposable C2 client. Capture each role's current digest/env/mounts +first (query `cca6355c`, remaining six `46a04c1e`) for exact rollback. No Rust, +ingestor, V1, Kafka topology/offset reset, cache reset or order-path action. +Then run the single replacement C2 from the packaged final image, with no +source patch mounts; use a new evidence directory and preserve prior failures. + +Final image result: `qdl-v2-python:2.0.12-35a7cd8`, digest +`sha256:1c1392bf636dc40c67cc73a2e5ea5e8d17f4e53ca4ecb8c62ac387be4262045a`, +passes packaged SDK30/30 and offline Trivy fixable HIGH/CRITICAL count0. +All eight approved Python roles now use that one image. Per-role env/mounts +are preserved; capture again reports44 protected services unchanged and +RUST_PRIMARY. Final C2 evidence target is `c2-release-final/`; no source mounts +or replay/cache resets are used. Release remains conditional on its result. + +Safe source cleanup inventory: Python310 SDK worktree and phase-D no-order +worktree are clean and ancestors of remote dev; the detached V1 test worktree +is also represented. None is referenced by any container mount or Compose +provenance. Remove only those after checking ignored files are disposable +bytecode. Retain phase-C reader worktree: it has four commits not represented +by dev (`cbb5161`, `9991ed2`, `2fb9146`, `70ac65b`), so deleting it would lose +unmerged development. Do not merge that unrelated projector branch into this +release just to claim a tidy directory. Active closure worktree stays until +publication and canonical-checkout synchronization are complete. + +Final C2 stopped during opening with BOOK_SNAPSHOT DATA_STALE for OKX ETH. +Its immediate typed status is LIVE/complete/no-gap, event age633ms and LIVE +session age806ms. The C2 retry predicate still requires NOT_APPLICABLE for +every book snapshot, contradicting the shared session-aware runtime contract. +This is a harness contract defect, not evidence that the rejected stale read +was usable. Correct only that predicate: a matching verified book may wait +within the existing deadline for a subsequent strict read when its session is +LIVE and within the declared SLA, or legitimately NOT_APPLICABLE without a +session requirement. Unknown/disconnected/stale session, mismatched identity, +gap, incomplete data and deadline exhaustion remain failures. Test the shared +matrix, not an ETH exception. No runtime image/build/recreate is needed for +this harness-only correction; use the final packaged SDK with the exact tested +harness file mounted read-only. Retain the failed receipt. Run the affected +book class together before the replacement full-scope acceptance. + +Cleanup completed: disposable Rust/scanner/C2 clients, the scanner/toolchain +images and exported test image archives removed; disk152G used/138G available +before scoped cleanup,149G/141G after. Removed two merged feature worktrees and +one clean detached V1 test checkout after no-runtime-reference verification. +The phase-C reader branch has four genuinely unmerged commits and is retained. +Shared BuildKit records and exact active/rollback images remain deliberately +retained; no broad prune or production data deletion was performed. + +Shared book-session correction: 83/83 receipt/identity/observation tests PASS +(29.758s), including40 venue/symbol/session-SLA/event-state combinations and +12 identity/quality/session rejection cases. Legacy sessionless snapshot is +preserved, absent required session and deadline exhaustion still reject. +Real read-only probe `book-class-verified/acceptance.json`: 10/10 books across +Binance/OKX, both replicas, 20/20 strict snapshot validations PASS; zero order +actions/provider-direct connections. Initial probe helper missing a required +factory argument made no reads and was corrected; its failed setup evidence +is retained. The first unit invocation also had one mistyped test-module name; +the complete corrected invocation above passed. Serving runtime remains35a7cd8. +Run the replacement C2 with only the tested harness file mounted read-only; +this is an auditable test-harness revision, not a serving-image substitution. + +Publication documentation check: the service-access guide still introduces +the V1 Redis client as its universal default. Clarify the existing versioned +V2 primary SDK/query/gRPC contract first and label the preserved V1 sections +as compatibility/fallback instructions. This changes no endpoint, entitlement, +consumer config or runtime. The release must not direct new crypto consumers +back to the legacy route by accident. Public certificate/notes will include +the actual image digests and tested scope, not a claim that every possible +symbol or VN market-hours execution has been certified. + +Replacement C2 milestone: `c2-session-book-final/` reports opening299/299 PASS +in881.495s with the quota policy unchanged. The subsequent300s observation +and batch closing revalidation are in progress; this is not a final PASS. +CI run33948049168 for harness commit92edc17 is fully green (unit, contract, +Python3.10 SDK). The real book-class probe took2.532s for20 strict reads, +slowest0.058s; all typed states LIVE. Service-guide route inventory matches +all11 current V2 HTTP router operations. Read-only capture still reports44 +protected services unchanged and RUST_PRIMARY. No additional image was built. + +C2 closing result: opening299/299 and300.100s observation PASS; closing failed +with `native BASIS cooldown exceeds the bounded Reference/L2 acceptance +deadline`. No release certificate is issued. The shared closing harness passes +the15s cache-read timeout as the deadline for all reference batches, although +reference requests declare60s and the approved overall closing bound is120s. +Propagate the one existing absolute closing deadline to every reference worker +instead of creating a shorter cache-read deadline. Keep per-request provider +deadlines, Rust cooldown/one-deferral policy and overall120s unchanged. Test +deadline propagation, cooldown beyond the bound, cancellation and both venues; +run the real closing/reference class as a bounded probe before any further +full replacement. No runtime/source-provider change, quota increase or image. + +The grouped closing probe completed in13.042s: monitoring4/4 and Binance +alpha125/125 PASS, including native BASIS on both replicas. Trading-System +and OKX alpha groups instead expose the remaining shared BAR harness error: +closing asks for a single row but the parity validator requires an immutable +overlap across one legitimate final-candle rollover. Keep two final rows for +the bounded closing read; compare the immutable overlap using the existing +one-row-rollover validator and still validate each current tail strictly. +No timestamp adjustment or arbitrary skew is allowed. Test exact/one-row +rollover and conflicting/disjoint/multi-row windows, then rerun only this +closing class. The overall acceptance and market-data contract stay unchanged. + +Closing-class corrections pass59/59 focused tests in10.570s (including native +60s deferral within the existing120s bound, rejection beyond the same bound, +two-row immutable overlap and unchanged strict tail quality). First reference +suite invocation had one mistyped module name; the corrected complete command +is `python -B -m unittest -q tests.test_phase105_native_basis_lane +tests.test_phase105_identity_acceptance tests.test_reference_l2_consumer_acceptance +tests.test_phase103_consumer_acceptance` inside the network-none existing image. +Real probe `closing-class-aligned/acceptance.json` is PASS across all4 consumers: +monitoring4, Trading-System60, Binance alpha125, OKX alpha110 =299/299 products. +It reads both query replicas, respects the existing provider admission lane, +and makes zero order calls. Both disposable closing-probe clients were removed. +Use the same final serving image with the exact three tested harness scripts +mounted read-only for the final replacement C2; no image build or runtime roll. + +Final C2 **PASS**:299 opening/closing products,234 durable +65 on-demand, +opening883.031s, observation300.100s, closing13.401s. Client/cursor removed; +zero order/provider-direct calls; protected44 services unchanged. Raw receipt +is `c2-deadline-final/acceptance.json`; it is immutable and must not be replaced. + +Offline certificate assembly exposed two evidence-mapping defects, not a +failed endpoint: the helper passed the normalized three-field V1 summary +instead of the original complete verified attestation, and the aggregator +uses opening quality and drops session/eligibility metadata for OBSERVE TRADE. +Correct the helper input and carry the existing typed session/complete/ +execution-eligibility evidence into release observations. A quiet TRADE or +BOOK_DELTA may be available only under declared OBSERVE + live bounded session, +complete/no-gap and non-executable quality; price/BLOCK/PAUSE contracts retain +their freshness bound. Missing session evidence cannot certify a stale feed. +Use a separately hashed, current full-scope closing read for freshness, linked +to the unchanged passing C2 for cursor/reconnect/fallback/300s proof. This is +offline certificate mapping plus no-order current reads, not another C2, +runtime change, freshness relaxation or new phase. Preserve failed assembly. + +Certificate mapping tests:104/104 release/identity/receipt regressions PASS +in51.703s; final age-binding subset26/26 PASS in30.842s. Tests cover the10-symbol +venue matrix for quiet TRADE/BOOK_DELTA, BLOCK/PAUSE/QUOTE/book-price rejection, +disconnected/expired/missing session, incomplete/gapped data, malformed fields, +current-read C2 digest/full-scope identity, duplicate/missing products, old or +expired evidence and immutable original C2. Counts overlap, not additive. +This code is offline certification only; no serving image is rebuilt/rolled. +Capture current closing observations using the tested mapping, bind their hash +to the immutable passing C2, and certify with the original full V1 attestation. + +Current-read helper initially stopped before any endpoint call: the immutable +host C2 receipt is mode0600 and the test client drops to UID10001. Preserve that +original and failed setup; mount an identical payload-free receipt copy with +read permission into the disposable client. No credentials or runtime file +permissions change. Capture label is `release-current-reads-verified/`. + +Current-read capture299/299 completed, but the stricter offline certificate +correctly rejects26 hot products whose receive age exceeded2-3s while the +mixed warmup batch finished. All reported source ages/session states remained +valid; do not discard receive age or bless executable stale data. Closing must +separate feed classes and cap hot batches at8 (one bounded local cache lane), +retaining the declared BAR batch limit, existing identity quotas and120s total +deadline. Verify exact scope/cardinality, class isolation and both replicas; +then recapture current reads only. Original full C2 remains immutable and PASS; +certificate remains BLOCKED until the strict source AND receive bounds pass. + +Grouped closing/certificate regressions42/42 PASS in22.310s, covering all10 +venue/symbol pairs and feed-isolated bounded batch/cardinality on both replicas. +The initial invocation also named a nonexistent extra test module; the exact +corrected invocation used `tests.test_phase105_identity_acceptance`, +`tests.test_release_session_observations`, `tests.test_phase105_release_observations`, +`tests.test_phase105_release_certification`. Container removed; no new image. + +**Final exit (2026-09-05 ~07:08 UTC): CERTIFIED.** Feed-isolated current reads +cover299/299 products. The offline certificate passes all5 existing gates: +release route readiness, V1 provenance, runtime handoff, V2 consumer acceptance, +V1 fallback-return. Result:299 V2_PRIMARY,4 explicitly excluded VN V1_PRIMARY, +0 blocked,0 active fallback,0 resource-budget violations. No SLA was changed. +Certificate SHA256 `0b67916ef677f7ee099de510a1731948b2eea85e876663ff99cf7d67e4bc1b2e`. +Public compact evidence and notes live in +[`upgrade/evidence/releases/v2.0.12/`](upgrade/evidence/releases/v2.0.12/). +The original full C2 and failed attempts stay immutable in the private evidence +root; current-read and harness/source digests are recorded separately. + +Scope: monitoring4, Trading-System paper adapter60, Binance alpha SDK125, +OKX alpha SDK110. There are234 durable and65 on-demand consumer products, +including150 BAR consumer requirements over140 physical final-BAR bindings, +24 TRADE,20 QUOTE,20 BOOK_SNAPSHOT,20 BOOK_DELTA,20 MARK_INDEX_PRICE and45 other +reference products. The full C2 proves opening883.031s, observation300.100s, +closing13.401s; quota pacing is included in opening duration, not endpoint +latency. Current closing capture uses117 millicores and215113728 bytes RSS; +full C2 uses69 millicores and256983040 bytes peak RSS. These are client metrics, +not whole-platform resource claims. Zero order/direct-provider client calls; +all44 protected services and Rust authority unchanged. Signed cursor/reconnect, +seven allowed V1 fallbacks and292 forbidden-fallback routes retain their proof. + +No further C2, image build or runtime roll is required by this closure. +Serving Python remains `qdl-v2-python:2.0.12-35a7cd8` / `sha256:1c1392bf...2045a` +for2 query,2 stream,3 projector and BAR edge; Rust remains +`qdl-v2-rust:2.0.12-d536098` / `sha256:36a822c0...4dce65`. +Source corrections after35a7cd8 affect certification/test harness only. +Runtime config is `phasec36-reference-l2-r14`, outside source under local-state. +V1 `sha256:dbfb5784...5d65`, Kafka offsets/topology, Redis, SQLite, Trading System, +alpha and order path were preserved. This certifies declared crypto data-plane +products on this host, not VN market-hours, broker execution or independent DR. + +Publication is separate from certification: reserve new tag `v2.0.12` (do not +move existing `v2.0.0`). Repo policy requires release PR dev -> main. No such PR +is open and host has SSH push but no authenticated GitHub API/CLI; owner was +asked for the release PR through the async channel. Push tested source/package +to dev, then the tag-triggered release workflow publishes notes/certificate +after the tag is on main. Do not describe an uncreated tag/release as published. + +Approved cleanup retention: keep active35a7cd8 plus the exact immediate rollback +set, query `cca6355c` (d7c7506) and other Python roles `46a04c1e` (c57658a), +with private per-role Compose overrides in `final-release-images/`. Retire only +the superseded, container-unreferenced images `08dd22c3`, `f6081050`, `bd0163fd`; +their old rollout packets are historical evidence, not the current rollback. +Keep Rust/V1/Trading-System images and all data/volumes/networks. Broad BuildKit +prune was not permitted and will not be substituted by an equivalent command. + +Cleanup guard stopped before deletion: `bd0163fd` is referenced by the exited +operational `stable_tls_init` and `stable_state_init` containers. Retain that +image/container pair as operational references, not disposable smoke. Revised +deletion set is only08dd22c3/f6081050; no force, no removal of init containers. + +Cleanup completed: two image IDs / three tags removed; available disk increased +from152239054848 to153267318784 bytes (+1028263936 bytes, about0.96GiB). +All container IDs/image/start/restart tuples are unchanged across cleanup; +read-only postcheck again confirms44 protected services unchanged and11V2 +roles healthy without OOM/restarts. Disposable C2/current-read clients and +cursor directories are absent. No volume, network or runtime data was removed. +BuildKit inventory57 records/5.801GB includes shared retained-layer cache; +broad prune was intentionally not performed. Earlier scoped builder/scanner +artifacts and three fully represented temporary worktrees were removed as +recorded above. Do not delete unrelated operational stopped containers. + +Source/release reconciliation: canonical checkout `/home/bobby/data_layer` +remains clean at1d933960 (`fix/phase54-l2-bootstrap`); runtime uses immutable +images and external r14 configuration, not that branch's source. Active closure +checkout is `/home/bobby/.worktrees/data-layer-dev-closure`, branch +`fix/l2-execution-readiness-closure`; tested source throughf0118d8 is on remote +dev. Keep it until release/main synchronization. The separate phase-C checkout +at70ac65b has four genuinely unmerged commits and is deliberately preserved. +Remote stable main remains9d6dfb9 and existingv2.0.0 remains7ebe1d1; newv2.0.12 +package is certified but not tagged/published before the required release PR. +This is the remaining publication boundary, not another market-data test gate. diff --git a/Dockerfile.phase8-rust b/Dockerfile.phase8-rust index 060e3b21..247d50ca 100644 --- a/Dockerfile.phase8-rust +++ b/Dockerfile.phase8-rust @@ -6,6 +6,7 @@ RUN apt-get update \ make pkg-config zlib1g-dev \ && rm -rf /var/lib/apt/lists/* RUN rustup component add rustfmt clippy +ENV CARGO_BUILD_JOBS=1 WORKDIR /src COPY Cargo.toml Cargo.lock ./ COPY deny.toml ./deny.toml diff --git a/config/v2/stable-acquisition-bindings.yaml b/config/v2/stable-acquisition-bindings.yaml index 67132b8a..41bcb0c4 100644 --- a/config/v2/stable-acquisition-bindings.yaml +++ b/config/v2/stable-acquisition-bindings.yaml @@ -157,6 +157,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: binance-usdm-bnbusdt-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -170,6 +171,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: binance-usdm-bnbusdt-quote mode: RUST_NATIVE runtime: BINANCE @@ -363,6 +365,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: binance-usdm-btcusdt-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -376,6 +379,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: binance-usdm-btcusdt-quote mode: RUST_NATIVE runtime: BINANCE @@ -517,6 +521,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: binance-usdm-dogeusdt-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -530,6 +535,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: binance-usdm-dogeusdt-quote mode: RUST_NATIVE runtime: BINANCE @@ -723,6 +729,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: binance-usdm-ethusdt-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -736,6 +743,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: binance-usdm-ethusdt-quote mode: RUST_NATIVE runtime: BINANCE @@ -877,6 +885,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: binance-usdm-solusdt-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -890,6 +899,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: binance-usdm-solusdt-quote mode: RUST_NATIVE runtime: BINANCE @@ -1194,6 +1204,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: okx-swap-bnb-usdt-swap-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1207,6 +1218,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: okx-swap-bnb-usdt-swap-quote mode: RUST_NATIVE runtime: OKX @@ -1340,6 +1352,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: okx-swap-btc-usdt-swap-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1353,6 +1366,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: okx-swap-btcusdt-bar-1m mode: RUST_NATIVE runtime: OKX @@ -1502,6 +1516,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: okx-swap-doge-usdt-swap-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1515,6 +1530,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: okx-swap-doge-usdt-swap-quote mode: RUST_NATIVE runtime: OKX @@ -1656,6 +1672,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: okx-swap-eth-usdt-swap-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1669,6 +1686,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: okx-swap-eth-usdt-swap-quote mode: RUST_NATIVE runtime: OKX @@ -1810,6 +1828,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: okx-swap-sol-usdt-swap-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1823,6 +1842,7 @@ bindings: depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + materialized_snapshot_interval_ms: 1000 - binding_id: okx-swap-sol-usdt-swap-quote mode: RUST_NATIVE runtime: OKX diff --git a/config/v2/stable-primary-consumer-routing.yaml b/config/v2/stable-primary-consumer-routing.yaml index d0cbe0e8..3e8705a2 100644 --- a/config/v2/stable-primary-consumer-routing.yaml +++ b/config/v2/stable-primary-consumer-routing.yaml @@ -1,5 +1,5 @@ schema: qdl.v2.shared-primary-consumer-route.v1 -revision: 3 +revision: 4 contract_version: 2.0.0 target_route: V2_PRIMARY rollback_route: V1 diff --git a/config/v2/stable-v2-release-routing.yaml b/config/v2/stable-v2-release-routing.yaml index d1828a54..56559fd8 100644 --- a/config/v2/stable-v2-release-routing.yaml +++ b/config/v2/stable-v2-release-routing.yaml @@ -1,5 +1,5 @@ schema: qdl.v2.stable-release-routing.v1 -revision: 14 +revision: 17 contract_version: 2.0.0 source_catalog: path: /app/config/v2/stable-source-bindings.yaml @@ -53,305 +53,889 @@ consumers: reason: V1_OKX_TRADE_EQUIVALENCE_UNPROVEN - consumer_id: alpha.binance.paper.stable manifest: /app/consumers/stable/alpha-binance-paper.yaml - manifest_revision: 8 - manifest_sha256: cebf10e4aa62596badb0ce661d49320bafaa8664f3547aea25665f32494ae2b5 + manifest_revision: 10 + manifest_sha256: 2a1d98de3900d360a50e095293964d67aba7a5085a0ae74251e37e86160bcbd3 demand_revision: 6 products: + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:TRADE::crypto_primary_v2 + route: V2_PRIMARY + fallback: V1 + reason: null + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:QUOTE::crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_QUOTE_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:1m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:3m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:5m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:15m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:30m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:1h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:2h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:4h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:6h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:8h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:12h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:1d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:3d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:1w:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BOOK_SNAPSHOT::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BOOK_DELTA::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:TRADE::crypto_primary_v2 route: V2_PRIMARY fallback: V1 reason: null + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:QUOTE::crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_QUOTE_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:1m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:3m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:5m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:15m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:30m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:1h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:2h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:4h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:6h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:8h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:12h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:1d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:3d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:1w:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BOOK_SNAPSHOT::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BOOK_DELTA::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:TRADE::crypto_primary_v2 + route: V2_PRIMARY + fallback: V1 + reason: null + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:QUOTE::crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_QUOTE_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:1m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:3m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:5m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:15m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:30m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:1h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:2h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:4h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:6h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:8h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:12h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:1d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:3d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:1w:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BOOK_SNAPSHOT::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BOOK_DELTA::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:TRADE::crypto_primary_v2 + route: V2_PRIMARY + fallback: V1 + reason: null + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:QUOTE::crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_QUOTE_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:1m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:3m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:5m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:15m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:30m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:1h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:2h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:4h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:6h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:8h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:12h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:1d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:3d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:1w:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BOOK_SNAPSHOT::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BOOK_DELTA::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:TRADE::crypto_primary_v2 route: V2_PRIMARY fallback: V1 reason: null - - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:1m:crypto_primary_v2 + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:QUOTE::crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_QUOTE_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:1m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:3m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:5m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:15m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:30m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:1h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:2h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:4h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:6h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:8h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:12h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:1d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:3d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:1w:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BOOK_SNAPSHOT::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BOOK_DELTA::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BASIS:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:CONTRACT_METADATA::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:FUNDING_RATE::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:LONG_SHORT_RATIO:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:MARK_INDEX_PRICE::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:OPEN_INTEREST:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:TAKER_FLOW:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BASIS:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:CONTRACT_METADATA::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:FUNDING_RATE::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:LONG_SHORT_RATIO:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:MARK_INDEX_PRICE::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:OPEN_INTEREST:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:TAKER_FLOW:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BASIS:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:CONTRACT_METADATA::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:FUNDING_RATE::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:LONG_SHORT_RATIO:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:MARK_INDEX_PRICE::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:OPEN_INTEREST:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:TAKER_FLOW:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BASIS:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:CONTRACT_METADATA::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:FUNDING_RATE::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:LONG_SHORT_RATIO:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:MARK_INDEX_PRICE::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:OPEN_INTEREST:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:TAKER_FLOW:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BASIS:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:CONTRACT_METADATA::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:FUNDING_RATE::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:LONG_SHORT_RATIO:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:MARK_INDEX_PRICE::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:OPEN_INTEREST:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:TAKER_FLOW:1d:crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN +- consumer_id: alpha.okx.paper.stable + manifest: /app/consumers/stable/alpha-okx-paper.yaml + manifest_revision: 9 + manifest_sha256: 5b30563bb7aa7345f5b2c7f030cfad2f10263a36a02f98f0a9847d2aef23a671 + demand_revision: 6 + products: + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:TRADE::crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_OKX_TRADE_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:QUOTE::crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_QUOTE_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:1m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_OKX_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:3m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:5m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:15m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:30m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:1h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:2h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:4h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:6h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:12h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:1d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:2d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:3d:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:1w:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BOOK_SNAPSHOT::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BOOK_DELTA::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:TRADE::crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_OKX_TRADE_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:QUOTE::crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_QUOTE_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:1m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_OKX_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:3m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:5m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:15m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:30m:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:1h:crypto_primary_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:2h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN - - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:1m:crypto_primary_v2 + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:4h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN - - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BAR:15m:crypto_primary_v2 + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:6h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED - - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BAR:15m:crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:12h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED - - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:TRADE::crypto_primary_v2 - route: V2_PRIMARY - fallback: V1 - reason: null - - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:1m:crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:1d:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN - - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BAR:15m:crypto_primary_v2 + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:2d:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED - - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:TRADE::crypto_primary_v2 - route: V2_PRIMARY - fallback: V1 - reason: null - - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:1m:crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:3d:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN - - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BAR:15m:crypto_primary_v2 + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:1w:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED - - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:TRADE::crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BOOK_SNAPSHOT::crypto_liquid_v2 route: V2_PRIMARY - fallback: V1 - reason: null - - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:1m:crypto_primary_v2 + fallback: BLOCKED + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BOOK_DELTA::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN - - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BAR:15m:crypto_primary_v2 + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:TRADE::crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED - - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:BASIS:1d:crypto_liquid_v2 + reason: V1_OKX_TRADE_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:QUOTE::crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:CONTRACT_METADATA::crypto_liquid_v2 + reason: V1_QUOTE_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:1m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:FUNDING_RATE::crypto_liquid_v2 + reason: V1_OKX_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:3m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:LONG_SHORT_RATIO:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:5m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:MARK_INDEX_PRICE::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:15m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:OPEN_INTEREST:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:30m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:TAKER_FLOW:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:1h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:BASIS:1d:crypto_liquid_v2 + reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:2h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:CONTRACT_METADATA::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:4h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:FUNDING_RATE::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:6h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:LONG_SHORT_RATIO:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:12h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:MARK_INDEX_PRICE::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:1d:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:OPEN_INTEREST:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:2d:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:TAKER_FLOW:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:3d:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:BASIS:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:1w:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:CONTRACT_METADATA::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BOOK_SNAPSHOT::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:FUNDING_RATE::crypto_liquid_v2 + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BOOK_DELTA::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:LONG_SHORT_RATIO:1d:crypto_liquid_v2 + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:TRADE::crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:MARK_INDEX_PRICE::crypto_liquid_v2 + reason: V1_OKX_TRADE_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:QUOTE::crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:OPEN_INTEREST:1d:crypto_liquid_v2 + reason: V1_QUOTE_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:1m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:TAKER_FLOW:1d:crypto_liquid_v2 + reason: V1_OKX_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:3m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:BASIS:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:5m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:CONTRACT_METADATA::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:15m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:FUNDING_RATE::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:30m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:LONG_SHORT_RATIO:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:1h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:MARK_INDEX_PRICE::crypto_liquid_v2 + reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:2h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:OPEN_INTEREST:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:4h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:TAKER_FLOW:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:6h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:BASIS:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:12h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:CONTRACT_METADATA::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:1d:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:FUNDING_RATE::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:2d:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:LONG_SHORT_RATIO:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:3d:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:MARK_INDEX_PRICE::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:1w:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:OPEN_INTEREST:1d:crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BOOK_SNAPSHOT::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:TAKER_FLOW:1d:crypto_liquid_v2 + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BOOK_DELTA::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN -- consumer_id: alpha.okx.paper.stable - manifest: /app/consumers/stable/alpha-okx-paper.yaml - manifest_revision: 7 - manifest_sha256: 049f38ca4c0445111a75b22f4f9d52455b374842d69c5f5af719c62d79f38f46 - demand_revision: 6 - products: + reason: V1_L2_EQUIVALENCE_UNPROVEN - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:TRADE::crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_OKX_TRADE_EQUIVALENCE_UNPROVEN - - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:TRADE::crypto_primary_v2 + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:QUOTE::crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_OKX_TRADE_EQUIVALENCE_UNPROVEN + reason: V1_QUOTE_EQUIVALENCE_UNPROVEN - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:1m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_OKX_BAR_EQUIVALENCE_UNPROVEN - - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:1m:crypto_primary_v2 - route: V2_PRIMARY - fallback: BLOCKED - reason: V1_OKX_BAR_EQUIVALENCE_UNPROVEN - - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:1h:crypto_primary_v2 + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:3m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED - - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:BAR:1h:crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:5m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED - - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:TRADE::crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:15m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_OKX_TRADE_EQUIVALENCE_UNPROVEN - - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:1m:crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:30m:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_OKX_BAR_EQUIVALENCE_UNPROVEN - - requirement_key: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de:BAR:1h:crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:1h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED - - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:TRADE::crypto_primary_v2 + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:2h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_OKX_TRADE_EQUIVALENCE_UNPROVEN - - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:1m:crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:4h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_OKX_BAR_EQUIVALENCE_UNPROVEN - - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:BAR:1h:crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:6h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED - - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:TRADE::crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:12h:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_OKX_TRADE_EQUIVALENCE_UNPROVEN - - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:1m:crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:1d:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_OKX_BAR_EQUIVALENCE_UNPROVEN - - requirement_key: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3:BAR:1h:crypto_primary_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:2d:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_PASS_THROUGH_PARITY_UNCERTIFIED - - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:CONTRACT_METADATA::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:3d:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:FUNDING_RATE::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BAR:1w:crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:MARK_INDEX_PRICE::crypto_liquid_v2 + reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BOOK_SNAPSHOT::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:OPEN_INTEREST::crypto_liquid_v2 + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:BOOK_DELTA::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED - reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:CONTRACT_METADATA::crypto_liquid_v2 + reason: V1_L2_EQUIVALENCE_UNPROVEN + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:CONTRACT_METADATA::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:FUNDING_RATE::crypto_liquid_v2 + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:FUNDING_RATE::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:MARK_INDEX_PRICE::crypto_liquid_v2 + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:MARK_INDEX_PRICE::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:OPEN_INTEREST::crypto_liquid_v2 + - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:OPEN_INTEREST::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN @@ -371,19 +955,19 @@ consumers: route: V2_PRIMARY fallback: BLOCKED reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:CONTRACT_METADATA::crypto_liquid_v2 + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:CONTRACT_METADATA::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:FUNDING_RATE::crypto_liquid_v2 + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:FUNDING_RATE::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:MARK_INDEX_PRICE::crypto_liquid_v2 + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:MARK_INDEX_PRICE::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:OPEN_INTEREST::crypto_liquid_v2 + - requirement_key: e49b54ae-c23d-5351-9e64-47934aac28f8:OPEN_INTEREST::crypto_liquid_v2 route: V2_PRIMARY fallback: BLOCKED reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN @@ -403,6 +987,22 @@ consumers: route: V2_PRIMARY fallback: BLOCKED reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:CONTRACT_METADATA::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:FUNDING_RATE::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:MARK_INDEX_PRICE::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN + - requirement_key: fb26214c-7b9b-5961-95b2-55154755af0f:OPEN_INTEREST::crypto_liquid_v2 + route: V2_PRIMARY + fallback: BLOCKED + reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - consumer_id: alpha.vn.paper.stable manifest: /app/consumers/stable/alpha-vn-paper.yaml manifest_revision: 1 @@ -419,18 +1019,18 @@ consumers: reason: VN_REAL_PROVIDER_GATE_UNEXERCISED - consumer_id: trading-system.paper.stable manifest: /app/consumers/stable/trading-system-paper.yaml - manifest_revision: 7 - manifest_sha256: c38b5044c6aad18a88e759e4c770a03753720f1badbff200893c90e0ab5ae132 + manifest_revision: 8 + manifest_sha256: a82ed9adf01401408a9b603d0bbb1e058e4573cff3be67c1b3caa3b7ddd6c0f7 demand_revision: 6 products: - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:TRADE::crypto_primary_v2 route: V2_PRIMARY - fallback: V1 - reason: null + fallback: BLOCKED + reason: V1_EXECUTION_SESSION_LIVENESS_UNPROVEN - requirement_key: ee93fabf-68df-5b50-8924-51bf25a5a757:TRADE::crypto_primary_v2 route: V2_PRIMARY - fallback: V1 - reason: null + fallback: BLOCKED + reason: V1_EXECUTION_SESSION_LIVENESS_UNPROVEN - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:QUOTE::crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED @@ -593,8 +1193,8 @@ consumers: reason: V1_L2_EQUIVALENCE_UNPROVEN - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:TRADE::crypto_primary_v2 route: V2_PRIMARY - fallback: V1 - reason: null + fallback: BLOCKED + reason: V1_EXECUTION_SESSION_LIVENESS_UNPROVEN - requirement_key: e4ce7249-a4e8-5073-beb8-dab908596c34:QUOTE::crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED @@ -605,8 +1205,8 @@ consumers: reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:TRADE::crypto_primary_v2 route: V2_PRIMARY - fallback: V1 - reason: null + fallback: BLOCKED + reason: V1_EXECUTION_SESSION_LIVENESS_UNPROVEN - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:QUOTE::crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED @@ -617,8 +1217,8 @@ consumers: reason: V1_FINAL_BAR_EQUIVALENCE_UNPROVEN - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:TRADE::crypto_primary_v2 route: V2_PRIMARY - fallback: V1 - reason: null + fallback: BLOCKED + reason: V1_EXECUTION_SESSION_LIVENESS_UNPROVEN - requirement_key: b2d78145-b541-58e2-941b-06c44c65e45f:QUOTE::crypto_primary_v2 route: V2_PRIMARY fallback: BLOCKED diff --git a/consumers/stable/alpha-binance-paper.yaml b/consumers/stable/alpha-binance-paper.yaml index e211c556..860ee01f 100644 --- a/consumers/stable/alpha-binance-paper.yaml +++ b/consumers/stable/alpha-binance-paper.yaml @@ -5,7 +5,7 @@ metadata: owner: alpha-platform subject: spiffe://qdl/paper/alpha-binance-stable environment: paper - revision: 8 + revision: 10 spec: sdk_major: 2 rollback_contract: V1 @@ -22,43 +22,1005 @@ spec: quotas: requests_per_minute: 180 max_batch_items: 50 - max_warmup_rows: 2000 + max_warmup_rows: 10000 max_streams: 20 max_buffer_events: 2000 requirements: + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: TRADE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 15000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: QUOTE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 5000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1m + warmup_limit: 10000 + max_freshness_ms: 180000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3m + warmup_limit: 10000 + max_freshness_ms: 540000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 5m + warmup_limit: 10000 + max_freshness_ms: 900000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 15m + warmup_limit: 10000 + max_freshness_ms: 2700000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 30m + warmup_limit: 10000 + max_freshness_ms: 5400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1h + warmup_limit: 10000 + max_freshness_ms: 10800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 2h + warmup_limit: 10000 + max_freshness_ms: 21600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 4h + warmup_limit: 10000 + max_freshness_ms: 43200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 6h + warmup_limit: 10000 + max_freshness_ms: 64800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 8h + warmup_limit: 10000 + max_freshness_ms: 86400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 12h + warmup_limit: 10000 + max_freshness_ms: 129600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1d + warmup_limit: 10000 + max_freshness_ms: 259200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3d + warmup_limit: 10000 + max_freshness_ms: 777600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1w + warmup_limit: 10000 + max_freshness_ms: 1814400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BOOK_SNAPSHOT + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 60000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: BOOK_DELTA + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 2000 + event_recency_policy: OBSERVE + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 feed: TRADE consumer_grade: ALPHA source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 15000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: QUOTE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 5000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1m + warmup_limit: 10000 + max_freshness_ms: 180000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3m + warmup_limit: 10000 + max_freshness_ms: 540000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 5m + warmup_limit: 10000 + max_freshness_ms: 900000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 15m + warmup_limit: 10000 + max_freshness_ms: 2700000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 30m + warmup_limit: 10000 + max_freshness_ms: 5400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1h + warmup_limit: 10000 + max_freshness_ms: 10800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 2h + warmup_limit: 10000 + max_freshness_ms: 21600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 4h + warmup_limit: 10000 + max_freshness_ms: 43200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 6h + warmup_limit: 10000 + max_freshness_ms: 64800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 8h + warmup_limit: 10000 + max_freshness_ms: 86400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 12h + warmup_limit: 10000 + max_freshness_ms: 129600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1d + warmup_limit: 10000 + max_freshness_ms: 259200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3d + warmup_limit: 10000 + max_freshness_ms: 777600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1w + warmup_limit: 10000 + max_freshness_ms: 1814400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BOOK_SNAPSHOT + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 60000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: BOOK_DELTA + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 2000 + event_recency_policy: OBSERVE + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: TRADE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 15000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: QUOTE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 5000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1m + warmup_limit: 10000 + max_freshness_ms: 180000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3m + warmup_limit: 10000 + max_freshness_ms: 540000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 5m + warmup_limit: 10000 + max_freshness_ms: 900000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 15m + warmup_limit: 10000 + max_freshness_ms: 2700000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 30m + warmup_limit: 10000 + max_freshness_ms: 5400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1h + warmup_limit: 10000 + max_freshness_ms: 10800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 2h + warmup_limit: 10000 + max_freshness_ms: 21600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 4h + warmup_limit: 10000 + max_freshness_ms: 43200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 6h + warmup_limit: 10000 + max_freshness_ms: 64800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 8h + warmup_limit: 10000 + max_freshness_ms: 86400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 12h + warmup_limit: 10000 + max_freshness_ms: 129600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1d + warmup_limit: 10000 + max_freshness_ms: 259200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3d + warmup_limit: 10000 + max_freshness_ms: 777600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1w + warmup_limit: 10000 + max_freshness_ms: 1814400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BOOK_SNAPSHOT + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 60000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: BOOK_DELTA + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 2000 + event_recency_policy: OBSERVE + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: TRADE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 15000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: QUOTE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 5000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1m + warmup_limit: 10000 + max_freshness_ms: 180000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3m + warmup_limit: 10000 + max_freshness_ms: 540000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 5m + warmup_limit: 10000 + max_freshness_ms: 900000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 15m + warmup_limit: 10000 + max_freshness_ms: 2700000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 30m + warmup_limit: 10000 + max_freshness_ms: 5400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1h + warmup_limit: 10000 + max_freshness_ms: 10800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 2h + warmup_limit: 10000 + max_freshness_ms: 21600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 4h + warmup_limit: 10000 + max_freshness_ms: 43200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 6h + warmup_limit: 10000 + max_freshness_ms: 64800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 8h + warmup_limit: 10000 + max_freshness_ms: 86400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 12h + warmup_limit: 10000 + max_freshness_ms: 129600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1d + warmup_limit: 10000 + max_freshness_ms: 259200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3d + warmup_limit: 10000 + max_freshness_ms: 777600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1w + warmup_limit: 10000 + max_freshness_ms: 1814400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BOOK_SNAPSHOT + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null warmup_limit: 0 - max_freshness_ms: 15000 - event_recency_policy: OBSERVE + max_freshness_ms: 60000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST max_session_liveness_ms: 45000 + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: BOOK_DELTA + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 2000 + event_recency_policy: OBSERVE require_full_coverage: true - require_final_bars: true + require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + max_session_liveness_ms: 45000 - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: TRADE consumer_grade: ALPHA source_policy_id: crypto_primary_v2 + interval: null warmup_limit: 0 max_freshness_ms: 15000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST event_recency_policy: OBSERVE max_session_liveness_ms: 45000 + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + feed: QUOTE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 5000 require_full_coverage: true - require_final_bars: true + require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST - - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + max_session_liveness_ms: 45000 + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 + interval: 1m + warmup_limit: 10000 max_freshness_ms: 180000 require_full_coverage: true require_final_bars: true @@ -66,167 +1028,205 @@ spec: gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: EMIT_REVISIONS - interval: 1m - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 180000 + interval: 3m + warmup_limit: 10000 + max_freshness_ms: 540000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: EMIT_REVISIONS - interval: 1m - - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 1080000 + interval: 5m + warmup_limit: 10000 + max_freshness_ms: 900000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK - recovery: FRESH_SNAPSHOT - bar_revision_policy: LATEST + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 interval: 15m + warmup_limit: 10000 + max_freshness_ms: 2700000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 1080000 + interval: 30m + warmup_limit: 10000 + max_freshness_ms: 5400000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK - recovery: FRESH_SNAPSHOT - bar_revision_policy: LATEST - interval: 15m - - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 - feed: TRADE + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 0 - max_freshness_ms: 15000 - event_recency_policy: OBSERVE - max_session_liveness_ms: 45000 + interval: 1h + warmup_limit: 10000 + max_freshness_ms: 10800000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY - bar_revision_policy: LATEST - - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 180000 + interval: 2h + warmup_limit: 10000 + max_freshness_ms: 21600000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: EMIT_REVISIONS - interval: 1m - - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 1080000 + interval: 4h + warmup_limit: 10000 + max_freshness_ms: 43200000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK - recovery: FRESH_SNAPSHOT - bar_revision_policy: LATEST - interval: 15m - - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a - feed: TRADE + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 0 - max_freshness_ms: 15000 - event_recency_policy: OBSERVE - max_session_liveness_ms: 45000 + interval: 6h + warmup_limit: 10000 + max_freshness_ms: 64800000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY - bar_revision_policy: LATEST - - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 180000 + interval: 8h + warmup_limit: 10000 + max_freshness_ms: 86400000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: EMIT_REVISIONS - interval: 1m - - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 1080000 + interval: 12h + warmup_limit: 10000 + max_freshness_ms: 129600000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK - recovery: FRESH_SNAPSHOT - bar_revision_policy: LATEST - interval: 15m - - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f - feed: TRADE + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 0 - max_freshness_ms: 15000 - event_recency_policy: OBSERVE - max_session_liveness_ms: 45000 + interval: 1d + warmup_limit: 10000 + max_freshness_ms: 259200000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY - bar_revision_policy: LATEST - - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 180000 + interval: 3d + warmup_limit: 10000 + max_freshness_ms: 777600000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: EMIT_REVISIONS - interval: 1m - - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 1080000 + interval: 1w + warmup_limit: 10000 + max_freshness_ms: 1814400000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK - recovery: FRESH_SNAPSHOT + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + feed: BOOK_SNAPSHOT + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 60000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST - interval: 15m - - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + max_session_liveness_ms: 45000 + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + feed: BOOK_DELTA + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 2000 + event_recency_policy: OBSERVE + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a feed: BASIS consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -239,7 +1239,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a feed: CONTRACT_METADATA consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -252,7 +1252,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a feed: FUNDING_RATE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -265,7 +1265,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a feed: LONG_SHORT_RATIO consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -278,7 +1278,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a feed: MARK_INDEX_PRICE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -291,7 +1291,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a feed: OPEN_INTEREST consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -304,7 +1304,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a feed: TAKER_FLOW consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -317,7 +1317,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 feed: BASIS consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -330,7 +1330,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 feed: CONTRACT_METADATA consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -343,7 +1343,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 feed: FUNDING_RATE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -356,7 +1356,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 feed: LONG_SHORT_RATIO consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -369,7 +1369,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 feed: MARK_INDEX_PRICE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -382,7 +1382,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 feed: OPEN_INTEREST consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -395,7 +1395,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 feed: TAKER_FLOW consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -408,7 +1408,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f feed: BASIS consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -421,7 +1421,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f feed: CONTRACT_METADATA consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -434,7 +1434,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f feed: FUNDING_RATE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -447,7 +1447,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f feed: LONG_SHORT_RATIO consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -460,7 +1460,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f feed: MARK_INDEX_PRICE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -473,7 +1473,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f feed: OPEN_INTEREST consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -486,7 +1486,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f feed: TAKER_FLOW consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -499,7 +1499,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 feed: BASIS consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -512,7 +1512,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 feed: CONTRACT_METADATA consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -525,7 +1525,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 feed: FUNDING_RATE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -538,7 +1538,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 feed: LONG_SHORT_RATIO consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -551,7 +1551,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 feed: MARK_INDEX_PRICE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -564,7 +1564,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 feed: OPEN_INTEREST consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -577,7 +1577,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 feed: TAKER_FLOW consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -590,7 +1590,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: BASIS consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -603,7 +1603,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: CONTRACT_METADATA consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -616,7 +1616,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: FUNDING_RATE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -629,7 +1629,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: LONG_SHORT_RATIO consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -642,7 +1642,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: MARK_INDEX_PRICE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -655,7 +1655,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: OPEN_INTEREST consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -668,7 +1668,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: TAKER_FLOW consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 diff --git a/consumers/stable/alpha-okx-paper.yaml b/consumers/stable/alpha-okx-paper.yaml index 3700ede3..cd72c62d 100644 --- a/consumers/stable/alpha-okx-paper.yaml +++ b/consumers/stable/alpha-okx-paper.yaml @@ -5,7 +5,7 @@ metadata: owner: alpha-platform subject: spiffe://qdl/paper/alpha-okx-stable environment: paper - revision: 7 + revision: 9 spec: sdk_major: 2 rollback_contract: V1 @@ -22,43 +22,1005 @@ spec: quotas: requests_per_minute: 180 max_batch_items: 50 - max_warmup_rows: 2000 + max_warmup_rows: 10000 max_streams: 20 max_buffer_events: 2000 requirements: - - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: TRADE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 15000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: QUOTE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 5000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1m + warmup_limit: 10000 + max_freshness_ms: 180000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3m + warmup_limit: 10000 + max_freshness_ms: 540000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 5m + warmup_limit: 10000 + max_freshness_ms: 900000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 15m + warmup_limit: 10000 + max_freshness_ms: 2700000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 30m + warmup_limit: 10000 + max_freshness_ms: 5400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1h + warmup_limit: 10000 + max_freshness_ms: 10800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 2h + warmup_limit: 10000 + max_freshness_ms: 21600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 4h + warmup_limit: 10000 + max_freshness_ms: 43200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 6h + warmup_limit: 10000 + max_freshness_ms: 64800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 12h + warmup_limit: 10000 + max_freshness_ms: 129600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1d + warmup_limit: 10000 + max_freshness_ms: 259200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 2d + warmup_limit: 10000 + max_freshness_ms: 518400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3d + warmup_limit: 10000 + max_freshness_ms: 777600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1w + warmup_limit: 10000 + max_freshness_ms: 1814400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BOOK_SNAPSHOT + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 60000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: BOOK_DELTA + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 2000 + event_recency_policy: OBSERVE + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: TRADE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 15000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: QUOTE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 5000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1m + warmup_limit: 10000 + max_freshness_ms: 180000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3m + warmup_limit: 10000 + max_freshness_ms: 540000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 5m + warmup_limit: 10000 + max_freshness_ms: 900000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 15m + warmup_limit: 10000 + max_freshness_ms: 2700000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 30m + warmup_limit: 10000 + max_freshness_ms: 5400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1h + warmup_limit: 10000 + max_freshness_ms: 10800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 2h + warmup_limit: 10000 + max_freshness_ms: 21600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 4h + warmup_limit: 10000 + max_freshness_ms: 43200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 6h + warmup_limit: 10000 + max_freshness_ms: 64800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 12h + warmup_limit: 10000 + max_freshness_ms: 129600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1d + warmup_limit: 10000 + max_freshness_ms: 259200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 2d + warmup_limit: 10000 + max_freshness_ms: 518400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3d + warmup_limit: 10000 + max_freshness_ms: 777600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1w + warmup_limit: 10000 + max_freshness_ms: 1814400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BOOK_SNAPSHOT + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 60000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: BOOK_DELTA + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 2000 + event_recency_policy: OBSERVE + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: TRADE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 15000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: QUOTE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 5000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1m + warmup_limit: 10000 + max_freshness_ms: 180000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3m + warmup_limit: 10000 + max_freshness_ms: 540000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 5m + warmup_limit: 10000 + max_freshness_ms: 900000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 15m + warmup_limit: 10000 + max_freshness_ms: 2700000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 30m + warmup_limit: 10000 + max_freshness_ms: 5400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1h + warmup_limit: 10000 + max_freshness_ms: 10800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 2h + warmup_limit: 10000 + max_freshness_ms: 21600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 4h + warmup_limit: 10000 + max_freshness_ms: 43200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 6h + warmup_limit: 10000 + max_freshness_ms: 64800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 12h + warmup_limit: 10000 + max_freshness_ms: 129600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1d + warmup_limit: 10000 + max_freshness_ms: 259200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 2d + warmup_limit: 10000 + max_freshness_ms: 518400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3d + warmup_limit: 10000 + max_freshness_ms: 777600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1w + warmup_limit: 10000 + max_freshness_ms: 1814400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BOOK_SNAPSHOT + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 60000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: BOOK_DELTA + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 2000 + event_recency_policy: OBSERVE + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 feed: TRADE consumer_grade: ALPHA source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 15000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: QUOTE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 5000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1m + warmup_limit: 10000 + max_freshness_ms: 180000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3m + warmup_limit: 10000 + max_freshness_ms: 540000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 5m + warmup_limit: 10000 + max_freshness_ms: 900000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 15m + warmup_limit: 10000 + max_freshness_ms: 2700000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 30m + warmup_limit: 10000 + max_freshness_ms: 5400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1h + warmup_limit: 10000 + max_freshness_ms: 10800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 2h + warmup_limit: 10000 + max_freshness_ms: 21600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 4h + warmup_limit: 10000 + max_freshness_ms: 43200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 6h + warmup_limit: 10000 + max_freshness_ms: 64800000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 12h + warmup_limit: 10000 + max_freshness_ms: 129600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1d + warmup_limit: 10000 + max_freshness_ms: 259200000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 2d + warmup_limit: 10000 + max_freshness_ms: 518400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 3d + warmup_limit: 10000 + max_freshness_ms: 777600000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1w + warmup_limit: 10000 + max_freshness_ms: 1814400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BOOK_SNAPSHOT + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null warmup_limit: 0 - max_freshness_ms: 15000 - event_recency_policy: OBSERVE + max_freshness_ms: 60000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST max_session_liveness_ms: 45000 + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: BOOK_DELTA + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 2000 + event_recency_policy: OBSERVE require_full_coverage: true - require_final_bars: true + require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST - - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + max_session_liveness_ms: 45000 + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: TRADE consumer_grade: ALPHA source_policy_id: crypto_primary_v2 + interval: null warmup_limit: 0 max_freshness_ms: 15000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST event_recency_policy: OBSERVE max_session_liveness_ms: 45000 + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f + feed: QUOTE + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 5000 require_full_coverage: true - require_final_bars: true + require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + max_session_liveness_ms: 45000 - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 + interval: 1m + warmup_limit: 10000 max_freshness_ms: 180000 require_full_coverage: true require_final_bars: true @@ -66,167 +1028,205 @@ spec: gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: EMIT_REVISIONS - interval: 1m - - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 180000 + interval: 3m + warmup_limit: 10000 + max_freshness_ms: 540000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: EMIT_REVISIONS - interval: 1m - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 3780000 + interval: 5m + warmup_limit: 10000 + max_freshness_ms: 900000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK - recovery: FRESH_SNAPSHOT - bar_revision_policy: LATEST - interval: 1h - - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 3780000 + interval: 15m + warmup_limit: 10000 + max_freshness_ms: 2700000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK - recovery: FRESH_SNAPSHOT - bar_revision_policy: LATEST - interval: 1h - - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de - feed: TRADE + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f + feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 0 - max_freshness_ms: 15000 - event_recency_policy: OBSERVE - max_session_liveness_ms: 45000 + interval: 30m + warmup_limit: 10000 + max_freshness_ms: 5400000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY - bar_revision_policy: LATEST - - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 180000 + interval: 1h + warmup_limit: 10000 + max_freshness_ms: 10800000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: EMIT_REVISIONS - interval: 1m - - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 3780000 + interval: 2h + warmup_limit: 10000 + max_freshness_ms: 21600000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK - recovery: FRESH_SNAPSHOT - bar_revision_policy: LATEST - interval: 1h - - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 - feed: TRADE + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f + feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 0 - max_freshness_ms: 15000 - event_recency_policy: OBSERVE - max_session_liveness_ms: 45000 + interval: 4h + warmup_limit: 10000 + max_freshness_ms: 43200000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY - bar_revision_policy: LATEST - - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 180000 + interval: 6h + warmup_limit: 10000 + max_freshness_ms: 64800000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: EMIT_REVISIONS - interval: 1m - - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 3780000 + interval: 12h + warmup_limit: 10000 + max_freshness_ms: 129600000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK - recovery: FRESH_SNAPSHOT - bar_revision_policy: LATEST - interval: 1h - - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 - feed: TRADE + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f + feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 0 - max_freshness_ms: 15000 - event_recency_policy: OBSERVE - max_session_liveness_ms: 45000 + interval: 1d + warmup_limit: 10000 + max_freshness_ms: 259200000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY - bar_revision_policy: LATEST - - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 180000 + interval: 2d + warmup_limit: 10000 + max_freshness_ms: 518400000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: EMIT_REVISIONS - interval: 1m - - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: BAR consumer_grade: ALPHA source_policy_id: crypto_primary_v2 - warmup_limit: 500 - max_freshness_ms: 3780000 + interval: 3d + warmup_limit: 10000 + max_freshness_ms: 777600000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK gap_policy: BLOCK - recovery: FRESH_SNAPSHOT + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f + feed: BAR + consumer_grade: ALPHA + source_policy_id: crypto_primary_v2 + interval: 1w + warmup_limit: 10000 + max_freshness_ms: 1814400000 + require_full_coverage: true + require_final_bars: true + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: EMIT_REVISIONS + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f + feed: BOOK_SNAPSHOT + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 60000 + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST - interval: 1h + max_session_liveness_ms: 45000 - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f + feed: BOOK_DELTA + consumer_grade: ALPHA + source_policy_id: crypto_liquid_v2 + interval: null + warmup_limit: 0 + max_freshness_ms: 2000 + event_recency_policy: OBSERVE + require_full_coverage: true + require_final_bars: false + stale_policy: BLOCK + gap_policy: BLOCK + recovery: SNAPSHOT_AND_REPLAY + bar_revision_policy: LATEST + max_session_liveness_ms: 45000 + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 feed: CONTRACT_METADATA consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -239,7 +1239,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 feed: FUNDING_RATE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -252,7 +1252,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 feed: MARK_INDEX_PRICE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -265,7 +1265,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f + - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 feed: OPEN_INTEREST consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -278,7 +1278,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de feed: CONTRACT_METADATA consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -291,7 +1291,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de feed: FUNDING_RATE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -304,7 +1304,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de feed: MARK_INDEX_PRICE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -317,7 +1317,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de feed: OPEN_INTEREST consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -330,7 +1330,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 feed: CONTRACT_METADATA consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -343,7 +1343,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 feed: FUNDING_RATE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -356,7 +1356,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 feed: MARK_INDEX_PRICE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -369,7 +1369,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 feed: OPEN_INTEREST consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -382,7 +1382,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 feed: CONTRACT_METADATA consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -395,7 +1395,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 feed: FUNDING_RATE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -408,7 +1408,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 feed: MARK_INDEX_PRICE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -421,7 +1421,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 feed: OPEN_INTEREST consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -434,7 +1434,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: CONTRACT_METADATA consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -447,7 +1447,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: FUNDING_RATE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -460,7 +1460,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: MARK_INDEX_PRICE consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 @@ -473,7 +1473,7 @@ spec: gap_policy: BLOCK recovery: FRESH_SNAPSHOT bar_revision_policy: LATEST - - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: OPEN_INTEREST consumer_grade: ALPHA source_policy_id: crypto_liquid_v2 diff --git a/consumers/stable/trading-system-paper.yaml b/consumers/stable/trading-system-paper.yaml index 0ba483cb..25dcfc89 100644 --- a/consumers/stable/trading-system-paper.yaml +++ b/consumers/stable/trading-system-paper.yaml @@ -5,7 +5,7 @@ metadata: owner: trading-platform subject: spiffe://qdl/paper/trading-system-stable environment: paper - revision: 7 + revision: 8 spec: sdk_major: 2 rollback_contract: V1 @@ -459,6 +459,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -495,6 +497,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -531,6 +535,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -567,6 +573,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -603,6 +611,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -639,6 +649,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -675,6 +687,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -711,6 +725,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -747,6 +763,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -783,6 +801,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK diff --git a/contracts/v2/openapi.snapshot.json b/contracts/v2/openapi.snapshot.json index 056908eb..26585b63 100644 --- a/contracts/v2/openapi.snapshot.json +++ b/contracts/v2/openapi.snapshot.json @@ -2319,7 +2319,7 @@ "anyOf": [ { "exclusiveMinimum": 0.0, - "maximum": 86400000.0, + "maximum": 1814400000.0, "type": "integer" }, { @@ -3012,7 +3012,7 @@ "anyOf": [ { "exclusiveMinimum": 0, - "maximum": 86400000, + "maximum": 1814400000, "type": "integer" }, { @@ -3536,7 +3536,7 @@ "anyOf": [ { "exclusiveMinimum": 0, - "maximum": 86400000, + "maximum": 1814400000, "type": "integer" }, { @@ -3743,7 +3743,7 @@ "anyOf": [ { "exclusiveMinimum": 0, - "maximum": 86400000, + "maximum": 1814400000, "type": "integer" }, { @@ -4035,7 +4035,7 @@ "anyOf": [ { "exclusiveMinimum": 0, - "maximum": 86400000, + "maximum": 1814400000, "type": "integer" }, { diff --git a/docker-compose.v2-stable.yml b/docker-compose.v2-stable.yml index 09601b1e..4f86f052 100644 --- a/docker-compose.v2-stable.yml +++ b/docker-compose.v2-stable.yml @@ -227,6 +227,8 @@ services: cp /source/query/* /stable-certs/query/ && cp /source/stream/* /stable-certs/stream/ && cp /source/authority-dispatcher/* /stable-certs/authority-dispatcher/ && + test -f /stable-certs/query/client-ca-bundle.crt || cp /stable-certs/query/ca.crt /stable-certs/query/client-ca-bundle.crt && + test -f /stable-certs/stream/client-ca-bundle.crt || cp /stable-certs/stream/ca.crt /stable-certs/stream/client-ca-bundle.crt && chown -R 10001:10001 /stable-certs && find /stable-certs -type f -exec chmod 0440 {} + restart: "no" @@ -263,6 +265,7 @@ services: QDL_STABLE_PASS_THROUGH_ENABLED: "true" QDL_STABLE_AUDIT_PATH: /var/lib/qdl-stable/runtime/stable-query-1-audit.jsonl QDL_STABLE_TLS_CA_FILE: /stable-certs/query/ca.crt + QDL_STABLE_TLS_CLIENT_CA_FILE: /stable-certs/query/client-ca-bundle.crt QDL_STABLE_TLS_CERT_FILE: /stable-certs/query/server.crt QDL_STABLE_TLS_KEY_FILE: /stable-certs/query/server.key depends_on: {stable_redis: {condition: service_healthy}, stable_state_init: {condition: service_completed_successfully}, stable_tls_init: {condition: service_completed_successfully}} @@ -286,15 +289,17 @@ services: QDL_STABLE_PASS_THROUGH_ENABLED: "true" QDL_STABLE_AUDIT_PATH: /var/lib/qdl-stable/runtime/stable-query-2-audit.jsonl QDL_STABLE_TLS_CA_FILE: /stable-certs/query/ca.crt + QDL_STABLE_TLS_CLIENT_CA_FILE: /stable-certs/query/client-ca-bundle.crt QDL_STABLE_TLS_CERT_FILE: /stable-certs/query/server.crt QDL_STABLE_TLS_KEY_FILE: /stable-certs/query/server.key depends_on: {stable_redis: {condition: service_healthy}, stable_state_init: {condition: service_completed_successfully}, stable_tls_init: {condition: service_completed_successfully}} + healthcheck: {test: [CMD, python, -c, "import ssl,urllib.request; c=ssl.create_default_context(cafile='/stable-certs/query/ca.crt'); c.load_cert_chain('/stable-certs/query/server.crt','/stable-certs/query/server.key'); urllib.request.urlopen('https://localhost:8200/health/ready',context=c,timeout=2)"], interval: 5s, timeout: 3s, retries: 20} stream_v2_active: <<: *python command: [python, -m, app.entrypoints.stream_v2_stable] networks: - stable_internal: {} + stable_internal: {aliases: [qdl-v2-stream-a]} stable_ingress: {} stable_consumer: {aliases: [qdl-v2-stream-a]} ports: ["127.0.0.1:18210:8200", "127.0.0.1:18220:8210"] @@ -306,15 +311,17 @@ services: QDL_STABLE_GRPC_PORT: "8210" QDL_STABLE_AUDIT_PATH: /var/lib/qdl-stable/runtime/stable-stream-active-audit.jsonl QDL_STABLE_TLS_CA_FILE: /stable-certs/stream/ca.crt + QDL_STABLE_TLS_CLIENT_CA_FILE: /stable-certs/stream/client-ca-bundle.crt QDL_STABLE_TLS_CERT_FILE: /stable-certs/stream/server.crt QDL_STABLE_TLS_KEY_FILE: /stable-certs/stream/server.key depends_on: {stable_redis: {condition: service_healthy}, stable_state_init: {condition: service_completed_successfully}, stable_tls_init: {condition: service_completed_successfully}} + healthcheck: {test: [CMD, python, -c, "import ssl,urllib.request; c=ssl.create_default_context(cafile='/stable-certs/stream/ca.crt'); c.load_cert_chain('/stable-certs/stream/server.crt','/stable-certs/stream/server.key'); urllib.request.urlopen('https://localhost:8200/health/dependencies',context=c,timeout=2)"], interval: 5s, timeout: 3s, retries: 20} stream_v2_passive: <<: *python command: [python, -m, app.entrypoints.stream_v2_stable] networks: - stable_internal: {} + stable_internal: {aliases: [qdl-v2-stream-b]} stable_ingress: {} stable_consumer: {aliases: [qdl-v2-stream-b]} ports: ["127.0.0.1:18211:8200", "127.0.0.1:18221:8210"] @@ -326,12 +333,15 @@ services: QDL_STABLE_GRPC_PORT: "8210" QDL_STABLE_AUDIT_PATH: /var/lib/qdl-stable/runtime/stable-stream-passive-audit.jsonl QDL_STABLE_TLS_CA_FILE: /stable-certs/stream/ca.crt + QDL_STABLE_TLS_CLIENT_CA_FILE: /stable-certs/stream/client-ca-bundle.crt QDL_STABLE_TLS_CERT_FILE: /stable-certs/stream/server.crt QDL_STABLE_TLS_KEY_FILE: /stable-certs/stream/server.key depends_on: {stable_redis: {condition: service_healthy}, stable_state_init: {condition: service_completed_successfully}, stable_tls_init: {condition: service_completed_successfully}} + healthcheck: {test: [CMD, python, -c, "import ssl,urllib.request; c=ssl.create_default_context(cafile='/stable-certs/stream/ca.crt'); c.load_cert_chain('/stable-certs/stream/server.crt','/stable-certs/stream/server.key'); urllib.request.urlopen('https://localhost:8200/health/dependencies',context=c,timeout=2)"], interval: 5s, timeout: 3s, retries: 20} projector_v2: <<: *python + mem_limit: 768m command: [python, -m, app.entrypoints.projector_v2_stable] environment: <<: *stable-env @@ -348,10 +358,12 @@ services: QDL_STABLE_TLS_CERT_FILE: /stable-certs/projector/client.crt QDL_STABLE_TLS_KEY_FILE: /stable-certs/projector/client.key QDL_STABLE_STREAM_INGEST_URLS_JSON: '["https://stream_v2_active:8200","https://stream_v2_passive:8200"]' - # Bound r11 catch-up inside this role's existing 512 MiB cgroup. Kafka - # retains pressure once the projector pauses canonical consumption. + # Keep the Kafka and Python working sets bounded while a shared durable + # cache catches up; every durable event remains Kafka-authoritative. QDL_STABLE_MAX_PENDING_RECORDS: "2048" QDL_STABLE_MAX_PENDING_BYTES: "33554432" + QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS: "512" + QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES: "8388608" volumes: - stable_state:/var/lib/qdl-stable - stable_tls:/stable-certs:ro @@ -368,6 +380,7 @@ services: projector_v2_2: <<: *python + mem_limit: 768m command: [python, -m, app.entrypoints.projector_v2_stable] environment: <<: *stable-env @@ -386,6 +399,8 @@ services: QDL_STABLE_STREAM_INGEST_URLS_JSON: '["https://stream_v2_active:8200","https://stream_v2_passive:8200"]' QDL_STABLE_MAX_PENDING_RECORDS: "2048" QDL_STABLE_MAX_PENDING_BYTES: "33554432" + QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS: "512" + QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES: "8388608" volumes: - stable_state:/var/lib/qdl-stable - stable_tls:/stable-certs:ro @@ -402,6 +417,7 @@ services: projector_v2_3: <<: *python + mem_limit: 768m command: [python, -m, app.entrypoints.projector_v2_stable] environment: <<: *stable-env @@ -420,6 +436,8 @@ services: QDL_STABLE_STREAM_INGEST_URLS_JSON: '["https://stream_v2_active:8200","https://stream_v2_passive:8200"]' QDL_STABLE_MAX_PENDING_RECORDS: "2048" QDL_STABLE_MAX_PENDING_BYTES: "33554432" + QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS: "512" + QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES: "8388608" volumes: - stable_state:/var/lib/qdl-stable - stable_tls:/stable-certs:ro @@ -665,14 +683,19 @@ services: <<: *stable-env QDL_STABLE_INSTANCE_ID: stable-binance-bar-edge QDL_STABLE_CONSUMER_GROUP: stable-binance-bar-edge - QDL_STABLE_ACQUISITION_BINDINGS: /app/config/v2/stable-acquisition-bindings.yaml + # The image defaults are valid for a standalone build. A sealed runtime + # projection may override both files together without changing service + # topology or losing the final-BAR ownership contract on recreate. + QDL_STABLE_SOURCE_BINDINGS: ${QDL_STABLE_SOURCE_BINDINGS:-/app/config/v2/stable-source-bindings.yaml} + QDL_STABLE_ACQUISITION_BINDINGS: ${QDL_STABLE_ACQUISITION_BINDINGS:-/app/config/v2/stable-acquisition-bindings.yaml} QDL_STABLE_RUNTIME_DIR: /runtime QDL_KAFKA_BOOTSTRAP_SERVERS: kafka1:9092,kafka2:9092,kafka3:9092 QDL_KAFKA_CERT_ROOT: /stable-certs/producer QDL_KAFKA_CLIENT_ID: qdl-v2-stable-binance-bars # Must cover the greatest durable final-BAR warmup declared by the # governed Trading System/alpha manifests. - QDL_STABLE_BAR_WARMUP_ROWS: "1000" + QDL_STABLE_BAR_WARMUP_ROWS: "10000" + QDL_STABLE_BAR_MAX_CATCHUP_ROWS: "10000" # A final row is provider-confirmed, not timer-confirmed. Start the # first bounded read promptly after the close, then retry that exact # target until it is durably ACKed. diff --git a/docs/runbooks/phase54-config-derived-reader-release.md b/docs/runbooks/phase54-config-derived-reader-release.md new file mode 100644 index 00000000..afcf7b22 --- /dev/null +++ b/docs/runbooks/phase54-config-derived-reader-release.md @@ -0,0 +1,144 @@ +# Config-Derived Reader Release + +## Purpose + +This runbook promotes a tested Data Layer `dev` revision into one canonical +Python reader image and a secret-free alpha binding release bundle. It does +not create data-plane demand, a new worker, a new identity, a provider +connection, a paper order, or a broker request. + +The bundle is consumed later by a temporary alpha no-order proof. Query and +stream remain shared services; binding files are not a reason to create a +container per alpha, symbol, interval, or venue. + +## Immutable Inputs + +Record the following before build: + +- Data Layer `dev` source SHA and `qdl_sdk` version. +- Execution Alpha inventory SHA generated from actual Compose/config. +- Data Layer catalog, reference manifest, release-routing and policy hashes. +- Existing reader image digest and its four exact role names. + +The candidate tag format is `qdl-v2-python:-`. +The Docker image ID, rather than a mutable tag, is the rollout coordinate. +Rollback is a complete immutable mapping for each of the four reader roles; +do not assume the previous query and stream roles share one image. + +## Build And Source Gates + +1. Build exactly one image from the canonical Phase C worktree with OCI + revision equal to the recorded source SHA. +2. Inspect the image: revision label, release label, non-root `10001:10001` + user and immutable image ID must match the packet. +3. Run the selected binding/release/reference/L2/query/stream test matrix from + that image using `--network none`, read-only root filesystem and tmpfs-only + test state. No source mount is permitted for the image provenance test. +4. Export the real alpha Compose/config inventory in a disposable no-network + container. Compile it twice against the candidate's source artifacts. The + inventory SHA, compilation SHA and every binding SHA must be identical on + the second run. + +## Bundle Layout + +Create a new `0700` directory outside the source checkout: + +```text +/home/bobby/.local/state/qdl-v2/releases/-/ + inventory.json + bindings/ + .binding.json + compilation-report.json + reader-image.override.yml + reader-rollback.override.yml + release-manifest.json +``` + +`release-manifest.json` contains hashes, IDs, roles, source revision and +provenance only. It must not contain a credential, key, token, provider +payload, cursor, database dump, Redis/Kafka state or log body. + +Bindings are intentionally public-to-their-declared-workload control-plane +artifacts: each `bindings/*.binding.json` is immutable mode `0444` so the +non-root alpha UID can parse its sealed route policy. All other bundle files +remain operator-only mode `0640`. A binding never carries TLS/JWT material. + +## Workload Identity Mounts + +Never make a protected source private key world-readable or run an alpha as the +host user. Before a real alpha handoff, materialize one distinct mount under +the governed state root with `scripts/materialize_alpha_v2_identity.py`: + +```text +/home/bobby/.local/state/qdl-v2/workload-identities// + tls/ # ca.crt, client.crt: 0444; client.key: 0400 + jwt/ # public.pem: 0444; private.key: 0400 + identity-manifest.json # operator-only hashes/modes, no secret bytes +``` + +The source TLS/JWT directories stay private and unchanged. The helper must run +as root only to set the declared immutable container UID/GID ownership (today +alpha `1000:1000`) and performs a staging-directory rename. Mount only `tls/` +and `jwt/` read-only into that alpha; do not mount the parent state root. The +alpha UID must parse its binding and read both private keys in an isolated +no-network container before any V2 provider or Gateway/Risk call. + +`reader-image.override.yml` changes all four services to the one candidate +image. `reader-rollback.override.yml` restores the exact pre-roll image for +each service, so either file changes only these service image fields: + +```text +query_v2_1 +query_v2_2 +stream_v2_active +stream_v2_passive +``` + +The manifest stores `rollback_images` with one immutable reference/ID pair per +role. A missing role, mutable reference, duplicate service, or candidate-equal +rollback ID fails bundle generation. It is appended as the final Compose +override from the canonical checkout `/home/bobby/data_layer`. Existing +security, TLS, authority and bar-edge +override files are retained only after a rendered `docker compose config +--quiet` proves they are still needed. A `/tmp` override or feature worktree +is never a production selector. + +## Rolling Packet + +Before each serial recreate, capture only bounded metadata: role name, image +ID, health, restart count, RSS, manifest checksum and query/stream lag. + +Recreate the two query roles first, then inspect the stream lease and recreate +the currently observed standby followed by the current lease holder: + +1. `query_v2_2` +2. `query_v2_1` +3. stream replica with `gateway_lease=STANDBY` +4. stream replica with `gateway_lease=READY` + +The query replicas must return `/health/ready` `200`. Stream replicas use a +cooperative lease: the required condition is exactly one `gateway_lease=READY` +and one `gateway_lease=STANDBY`, with every non-lease dependency ready. A +standby deliberately returns `503` from `/health/ready`; inspect its +`/health/dependencies` instead. Lease ownership may move during the serial +roll, so the container name is not an authority assertion. The next role starts +only when this condition holds, every changed role has no restart/OOM state, +all four report the expected image/config generation and stay within their +existing resource limits. If one role fails, stop the sequence and recreate +only that role with the recorded old image and selector set. + +V1, Kafka topology/offsets, Redis, SQLite, Rust core, ingestors, bar edge, +projectors, Trading System, alpha services and order paths are excluded. + +## Exit And Cleanup + +Phase C passes only when the canonical image/bundle provenance is sealed, both +query replicas are ready, the stream pair holds exactly one clean leader and +one clean standby on the new image, and their original phase-name worktree or +temporary image override is not part of the live Compose command. The +subsequent Phase D alpha proof uses real Binance and OKX data but remains +no-order. + +Retain the active reader image and one named rollback image. Remove only +disposable test containers, temporary no-order state and unreferenced test +images after reachability checks; record disk before/after in the main plan. diff --git a/poetry.lock b/poetry.lock index 96ba2706..57f1670e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2415,6 +2415,64 @@ files = [ doc = ["reno", "sphinx"] test = ["pytest", "tornado (>=4.5)", "typeguard"] +[[package]] +name = "tomli" +version = "2.4.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.10\"" +files = [ + {file = "tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30"}, + {file = "tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a"}, + {file = "tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076"}, + {file = "tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9"}, + {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c"}, + {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc"}, + {file = "tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049"}, + {file = "tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e"}, + {file = "tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece"}, + {file = "tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a"}, + {file = "tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085"}, + {file = "tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9"}, + {file = "tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5"}, + {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585"}, + {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1"}, + {file = "tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917"}, + {file = "tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9"}, + {file = "tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257"}, + {file = "tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54"}, + {file = "tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a"}, + {file = "tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897"}, + {file = "tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f"}, + {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d"}, + {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5"}, + {file = "tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd"}, + {file = "tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36"}, + {file = "tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd"}, + {file = "tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf"}, + {file = "tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac"}, + {file = "tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662"}, + {file = "tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853"}, + {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15"}, + {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba"}, + {file = "tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6"}, + {file = "tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7"}, + {file = "tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232"}, + {file = "tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4"}, + {file = "tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c"}, + {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d"}, + {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41"}, + {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c"}, + {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f"}, + {file = "tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8"}, + {file = "tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26"}, + {file = "tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396"}, + {file = "tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe"}, + {file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"}, +] + [[package]] name = "typing-extensions" version = "4.15.0" @@ -2938,4 +2996,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "f5d8c82d042f5dd6bd468543bb5d7f457532c2528daba5e77415d50d45c2bca2" +content-hash = "314c2146b1f37132fb355d47a05654b7ac385e49852f581b36363bef0f62a724" diff --git a/pyproject.toml b/pyproject.toml index f4c627c4..485c81d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,8 @@ dependencies = [ "pillow (>=12.3.0,<13.0.0)", "pyjwt[crypto] (>=2.13.0,<3.0.0)", "soupsieve (>=2.8.4,<3.0.0)", - "starlette (>=1.3.1,<2.0.0)" + "starlette (>=1.3.1,<2.0.0)", + "tomli (>=2.0.1,<3.0.0) ; python_version < '3.11'" ] [build-system] diff --git a/qdl/_compat.py b/qdl/_compat.py new file mode 100644 index 00000000..39278eb9 --- /dev/null +++ b/qdl/_compat.py @@ -0,0 +1,27 @@ +"""Small Python-version compatibility primitives used by the internal core.""" + +from __future__ import annotations + +from enum import Enum + +try: # Python 3.11+ + from enum import StrEnum as StrEnum +except ImportError: # Python 3.10 remains a supported SDK/compiler runtime. + class StrEnum(str, Enum): + """Backport the observable behavior needed from :class:`enum.StrEnum`.""" + + def __str__(self) -> str: + return str(self.value) + + @staticmethod + def _generate_next_value_( + name: str, + start: int, + count: int, + last_values: list[object], + ) -> str: + del start, count, last_values + return name.lower() + + +__all__ = ["StrEnum"] diff --git a/qdl/adapters/binance/bar_edge.py b/qdl/adapters/binance/bar_edge.py index 53efae18..cdfc9db0 100644 --- a/qdl/adapters/binance/bar_edge.py +++ b/qdl/adapters/binance/bar_edge.py @@ -12,6 +12,10 @@ from qdl.raw.capture import capture_exact_frame +_MAX_HISTORY_ROWS = 10_000 +_HISTORY_PAGE_ROWS = 1_000 + + @dataclass(frozen=True) class BinanceBarRawBinding: market: str @@ -66,7 +70,7 @@ def _interval_ms(interval: str) -> int: def _fetch_rows( binding: BinanceBarRawBinding, *, - observed_ms: int, + end_time_ms: int, limit: int, attempts: int, fetcher: Callable, @@ -82,7 +86,7 @@ def _fetch_rows( binding.native_symbol, interval=binding.interval, limit=limit, - end_time=observed_ms, + end_time=end_time_ms, market=binding.market.lower(), ) break @@ -176,29 +180,66 @@ def fetch_closed_bar_history_raw_envelopes( sleep: Callable[[float], None] = time.sleep, test_provenance: bool = False, ) -> tuple[raw_provider_pb2.RawProviderEnvelope, ...]: - if limit < 1 or limit > 1000: - raise ValueError("Binance history limit must be between 1 and 1000") + if limit < 1 or limit > _MAX_HISTORY_ROWS: + raise ValueError( + f"Binance history limit must be between 1 and {_MAX_HISTORY_ROWS}" + ) observed_ms = int(now_ms if now_ms is not None else time.time() * 1000) interval_ms = _interval_ms(binding.interval) - # Binance caps a kline response at 1000 rows. Asking through wall-clock - # `now` can consume one slot with the still-open candle, yielding only 999 - # usable closed rows. The history API's cutoff is therefore the exclusive - # boundary of the current interval, not request observation time. - provider_end_ms = latest_closed_boundary_ms( + # Each Binance REST page is capped at 1,000 rows. The cursor starts at the + # latest known closed boundary and walks backward one complete page at a + # time; every provider response must stay within its requested end bound. + # This keeps a 10,000-row warmup exact rather than accepting duplicate or + # shifted pages when a vendor ignores a cursor. + page_end_ms = latest_closed_boundary_ms( binding.interval, observed_ms, provider="BINANCE", ) - 1 - rows = _fetch_rows( - binding, - observed_ms=provider_end_ms, - limit=min(1000, limit + 2), - attempts=attempts, - fetcher=fetcher, - sleep=sleep, - ) - closed = _closed_rows(rows, observed_ms=observed_ms, interval_ms=interval_ms) - selected = closed[-limit:] + remaining = limit + selected_by_open: dict[int, list] = {} + while remaining: + page_limit = min(_HISTORY_PAGE_ROWS, remaining) + rows = _fetch_rows( + binding, + end_time_ms=page_end_ms, + limit=page_limit, + attempts=attempts, + fetcher=fetcher, + sleep=sleep, + ) + closed = _closed_rows( + rows, + observed_ms=observed_ms, + interval_ms=interval_ms, + ) + if len(closed) != page_limit: + raise RuntimeError( + "Binance closed-bar history page is incomplete " + f"requested={page_limit} observed={len(closed)}" + ) + if any(int(row[6]) > page_end_ms for row in closed): + raise RuntimeError( + "Binance closed-bar history page exceeds requested end boundary" + ) + for row in closed: + open_time = int(row[0]) + previous = selected_by_open.get(open_time) + if previous is not None and previous != row: + raise RuntimeError( + "Binance history pages conflict for one open time" + ) + selected_by_open[open_time] = row + earliest_open_ms = min(int(row[0]) for row in closed) + next_page_end_ms = earliest_open_ms - 1 + if next_page_end_ms >= page_end_ms: + raise RuntimeError( + "Binance history pagination made no backward progress" + ) + page_end_ms = next_page_end_ms + remaining -= page_limit + + selected = tuple(selected_by_open[key] for key in sorted(selected_by_open)) if len(selected) != limit: raise RuntimeError( f"Binance closed-bar history is incomplete requested={limit} observed={len(selected)}" @@ -232,7 +273,7 @@ def fetch_latest_closed_bar_raw_envelope( interval_ms = _interval_ms(binding.interval) rows = _fetch_rows( binding, - observed_ms=observed_ms, + end_time_ms=observed_ms, limit=3, attempts=attempts, fetcher=fetcher, diff --git a/qdl/adapters/binance/reference.py b/qdl/adapters/binance/reference.py index a79b4e8d..d81843ce 100644 --- a/qdl/adapters/binance/reference.py +++ b/qdl/adapters/binance/reference.py @@ -723,9 +723,11 @@ async def _paginate( while pages < request.max_pages and len(selected) < request.limit: logical_start = cursor_start if direction == "FORWARD" else request.start_ms logical_end = request.end_ms if direction == "FORWARD" else cursor_end + provider_start = logical_start + provider_end = logical_end + boundary_tolerance_ms response = await page( - logical_start + provider_time_offset_ms, - logical_end + provider_time_offset_ms, + provider_start + provider_time_offset_ms, + provider_end + provider_time_offset_ms, min(per_page, request.limit - len(selected)), ) data = self._response_data(response, endpoint) @@ -740,15 +742,21 @@ async def _paginate( if not isinstance(row, Mapping): raise ReferenceProviderError(f"Binance {endpoint} history row has invalid shape") page_observations.append(parser(row)) - page_times = [item.observed_at_ns // 1_000_000 for item in page_observations] + # Some Binance history products identify a completed sampling period + # by its provider-side open timestamp but expose the canonical + # observation at period close for freshness. Pagination and + # coverage must remain in the provider's logical open-time domain. + page_times = [self._pagination_time_ms(item) for item in page_observations] oldest = min(page_times) newest = max(page_times) if direction == "FORWARD" and newest < cursor_start: raise ReferenceProviderError(f"Binance {endpoint} forward pagination made no progress") if direction == "BACKWARD" and oldest > cursor_end: raise ReferenceProviderError(f"Binance {endpoint} backward pagination made no progress") + lower_selection_bound = request.start_ms + upper_selection_bound = request.end_ms + boundary_tolerance_ms for item, timestamp_ms in zip(page_observations, page_times): - if request.start_ms <= timestamp_ms <= request.end_ms: + if lower_selection_bound <= timestamp_ms <= upper_selection_bound: prior = selected.get(timestamp_ms) if prior is not None and prior != item: raise ReferenceProviderError( @@ -935,7 +943,18 @@ def _taker_observation( ) if not fields: raise ReferenceProviderError("Binance taker-flow row has no numeric fields") - return self._observation(request, self._timestamp_ns(row, "timestamp"), fields) + period_open_ms = self._timestamp_ns(row, "timestamp") // 1_000_000 + period_close_ms = period_open_ms + canonical_interval_ms(request.interval or "") - 1 + return self._observation( + request, + period_close_ms * 1_000_000, + fields, + labels=( + ("period_open_time_ms", str(period_open_ms)), + ("period_close_time_ms", str(period_close_ms)), + ("timestamp_origin", "PROVIDER_PERIOD_START"), + ), + ) def _mark_index_observation( self, request: ReferenceRequest, row: Mapping[str, Any], received_at_ns: int @@ -1008,14 +1027,19 @@ def _basis_observation( ) if not fields: raise ReferenceProviderError("Binance basis row has no numeric fields") + period_open_ms = self._timestamp_ns(row, "timestamp") // 1_000_000 + period_close_ms = period_open_ms + canonical_interval_ms(request.interval or "") - 1 return self._observation( request, - self._timestamp_ns(row, "timestamp"), + period_close_ms * 1_000_000, fields, labels=( ("basis_series", request.basis_series.value), ("contract_selector", expected_contract), ("pair", pair), + ("period_open_time_ms", str(period_open_ms)), + ("period_close_time_ms", str(period_close_ms)), + ("timestamp_origin", "PROVIDER_PERIOD_START"), ), ) @@ -1104,6 +1128,27 @@ def _timestamp_ns_optional(row: Mapping[str, Any], key: str) -> int | None: raise ReferenceProviderError(f"Binance reference timestamp {key} must be positive") return timestamp_ms * 1_000_000 + @staticmethod + def _pagination_time_ms(observation: ReferenceObservation) -> int: + """Return the provider's logical history coordinate for pagination. + + Period-start products keep their raw coordinate in lineage labels while + ``observed_at_ns`` represents the completed period close used by V2 + freshness checks. Other products continue to paginate by observation + timestamp. + """ + + period_open = dict(observation.labels).get("period_open_time_ms") + if period_open is None: + return observation.observed_at_ns // 1_000_000 + try: + period_open_ms = int(period_open) + except ValueError as error: + raise ReferenceProviderError("Binance reference period-open label is invalid") from error + if period_open_ms <= 0: + raise ReferenceProviderError("Binance reference period-open label must be positive") + return period_open_ms + @staticmethod def _filter_value(row: Mapping[str, Any], filter_type: str, key: str) -> object | None: filters = row.get("filters") diff --git a/qdl/api_v2/models.py b/qdl/api_v2/models.py index 30f4fa0a..004fd55f 100644 --- a/qdl/api_v2/models.py +++ b/qdl/api_v2/models.py @@ -48,6 +48,12 @@ from qdl_sdk.reference import ReferenceBatchRequest, ReferenceBatchResponse +# The sealed V2 final-BAR catalog currently reaches a one-week interval with +# a 21-day freshness allowance. Keep the public boundary aligned with that +# governed policy while individual consumer manifests remain authoritative. +MAX_REQUIREMENT_FRESHNESS_MS = 21 * 86_400_000 + + class RequirementModel(ClosedModel): instrument_uid: str = Field(min_length=1, max_length=200) feed: FeedType @@ -55,7 +61,11 @@ class RequirementModel(ClosedModel): source_policy_id: str = Field(min_length=1, max_length=200) interval: str | None = Field(default=None, max_length=20) warmup_limit: int = Field(default=0, ge=0, le=10_000) - max_freshness_ms: int | None = Field(default=None, gt=0, le=86_400_000) + max_freshness_ms: int | None = Field( + default=None, + gt=0, + le=MAX_REQUIREMENT_FRESHNESS_MS, + ) event_recency_policy: StalePolicy | None = None max_session_liveness_ms: int | None = Field(default=None, gt=0, le=86_400_000) require_full_coverage: bool = True diff --git a/qdl/api_v2/router.py b/qdl/api_v2/router.py index 6da0e21c..33231585 100644 --- a/qdl/api_v2/router.py +++ b/qdl/api_v2/router.py @@ -20,6 +20,7 @@ InstrumentPageResponse, InstrumentResponse, MarketDataView, + MAX_REQUIREMENT_FRESHNESS_MS, ProblemDetails, QualityView, ReadinessItemResponse, @@ -605,7 +606,9 @@ async def snapshot( source_policy_id: str, consumer_grade: ConsumerGrade = ConsumerGrade.ALPHA, interval: str | None = None, - max_freshness_ms: int | None = Query(None, gt=0, le=86_400_000), + max_freshness_ms: int | None = Query( + None, gt=0, le=MAX_REQUIREMENT_FRESHNESS_MS + ), event_recency_policy: StalePolicy | None = None, max_session_liveness_ms: int | None = Query(None, gt=0, le=86_400_000), require_full_coverage: bool = True, @@ -653,7 +656,9 @@ async def warmup( ), max_cache_age_ms: int = Query(60_000, ge=0, le=86_400_000), deadline_ms: int = Query(20_000, ge=100, le=120_000), - max_freshness_ms: int | None = Query(None, gt=0, le=86_400_000), + max_freshness_ms: int | None = Query( + None, gt=0, le=MAX_REQUIREMENT_FRESHNESS_MS + ), event_recency_policy: StalePolicy | None = None, max_session_liveness_ms: int | None = Query(None, gt=0, le=86_400_000), require_full_coverage: bool = True, @@ -718,7 +723,9 @@ async def history( ), max_cache_age_ms: int = Query(60_000, ge=0, le=86_400_000), deadline_ms: int = Query(20_000, ge=100, le=120_000), - max_freshness_ms: int | None = Query(None, gt=0, le=86_400_000), + max_freshness_ms: int | None = Query( + None, gt=0, le=MAX_REQUIREMENT_FRESHNESS_MS + ), event_recency_policy: StalePolicy | None = None, max_session_liveness_ms: int | None = Query(None, gt=0, le=86_400_000), require_full_coverage: bool = True, @@ -885,7 +892,9 @@ async def feed_status( source_policy_id: str, consumer_grade: ConsumerGrade = ConsumerGrade.ALPHA, interval: str | None = None, - max_freshness_ms: int | None = Query(None, gt=0, le=86_400_000), + max_freshness_ms: int | None = Query( + None, gt=0, le=MAX_REQUIREMENT_FRESHNESS_MS + ), event_recency_policy: StalePolicy | None = None, max_session_liveness_ms: int | None = Query(None, gt=0, le=86_400_000), require_full_coverage: bool = True, diff --git a/qdl/canary/phase9.py b/qdl/canary/phase9.py index 40fd6c19..13750491 100644 --- a/qdl/canary/phase9.py +++ b/qdl/canary/phase9.py @@ -2,7 +2,7 @@ import uuid from dataclasses import dataclass -from enum import StrEnum +from qdl._compat import StrEnum from typing import Any, Mapping from qdl.certification.prerequisites import CandidateSlice diff --git a/qdl/canary/phase92.py b/qdl/canary/phase92.py index 92fc20f9..0c5eec77 100644 --- a/qdl/canary/phase92.py +++ b/qdl/canary/phase92.py @@ -4,7 +4,7 @@ import json import uuid from dataclasses import asdict, dataclass -from enum import StrEnum +from qdl._compat import StrEnum from typing import Any, Mapping from qdl.canary.phase9 import ProductionCanaryAuthorizer diff --git a/qdl/canary/phase93.py b/qdl/canary/phase93.py index 6b536f8f..b4039474 100644 --- a/qdl/canary/phase93.py +++ b/qdl/canary/phase93.py @@ -4,7 +4,7 @@ import json import uuid from dataclasses import asdict, dataclass -from enum import StrEnum +from qdl._compat import StrEnum from typing import Any, Mapping from qdl.canary.phase9 import ProductionCanaryAuthorizer diff --git a/qdl/certification/phase103_consumer_acceptance.py b/qdl/certification/phase103_consumer_acceptance.py index 660e747c..98766b74 100644 --- a/qdl/certification/phase103_consumer_acceptance.py +++ b/qdl/certification/phase103_consumer_acceptance.py @@ -12,7 +12,7 @@ import json from dataclasses import dataclass from decimal import Decimal, InvalidOperation -from enum import StrEnum +from qdl._compat import StrEnum from pathlib import Path from typing import Callable, Iterable, Sequence @@ -564,12 +564,16 @@ def validate_product_view( view, *, require_current_quality: bool = True, + state_replay: bool = False, ) -> None: """Validate one governed view. Historical BAR warmup rows retain identity, provenance, gap and payload checks, but cannot individually satisfy a *current* freshness SLA. The latest closed BAR is the row that carries current/live/execution quality. + A state-replay frame is likewise checked for identity, provenance, gap and + payload validity only; it can never certify current provider-session or + execution eligibility and must be followed by a strict current read-back. """ if ( view.instrument_uid != product.instrument_uid @@ -581,28 +585,37 @@ def validate_product_view( raise ValueError("V2 receipt identity, feed, interval or policy mismatches demand") if view.quality.gap_open or not view.quality.complete: raise ValueError("V2 receipt has an unresolved gap or incomplete coverage") + if state_replay and require_current_quality: + raise ValueError("V2 state replay cannot claim current execution quality") requirement = product.requirement max_freshness_ms = requirement.max_freshness_ms - observed_quiet_trade = ( - product.feed is FeedType.TRADE + # A quiet TRADE or BOOK_DELTA channel says nothing about the health of its + # provider session. The latter is admitted only through an explicit + # session SLA. BOOK_DELTA remains non-price continuity/replay evidence: + # callers must use a fresh BOOK_SNAPSHOT/QUOTE/MARK read to choose a price. + observed_quiet_continuity = ( + product.feed in {FeedType.TRADE, FeedType.BOOK_DELTA} and requirement.effective_event_recency_policy is StalePolicy.OBSERVE and view.quality.event_recency_state == "STALE" and view.quality.provider_session_state == "LIVE" ) - if ( - require_current_quality - and max_freshness_ms is not None - and view.quality.freshness_ms > max_freshness_ms - and not observed_quiet_trade - ): - raise ValueError("V2 receipt exceeds the governed freshness bound") - if requirement.max_session_liveness_ms is not None and ( + # The session is the authority for an observed quiet continuity channel. + # Check it first so a disconnected feed cannot be reported merely as an + # old last event. + if not state_replay and requirement.max_session_liveness_ms is not None and ( view.quality.provider_session_state != "LIVE" or view.quality.provider_session_liveness_ms is None or view.quality.provider_session_liveness_ms > requirement.max_session_liveness_ms ): raise ValueError("V2 receipt provider session liveness differs from demand") + if ( + require_current_quality + and max_freshness_ms is not None + and view.quality.freshness_ms > max_freshness_ms + and not observed_quiet_continuity + ): + raise ValueError("V2 receipt exceeds the governed freshness bound") if product.delivery is DeliveryClass.DURABLE: if ( view.source.venue != product.venue @@ -616,7 +629,7 @@ def validate_product_view( require_current_quality and requirement.consumer_grade.value == "EXECUTION" and not view.quality.execution_eligible - and not observed_quiet_trade + and not observed_quiet_continuity ): raise ValueError("execution-grade durable V2 receipt is not eligible") elif product.delivery is DeliveryClass.PROVIDER_PASS_THROUGH: diff --git a/qdl/certification/phase105_handoff.py b/qdl/certification/phase105_handoff.py index ee4aaa5d..62deab1c 100644 --- a/qdl/certification/phase105_handoff.py +++ b/qdl/certification/phase105_handoff.py @@ -12,6 +12,17 @@ V1_FALLBACK_COMMIT = "2b0dcf74454c9f87c352d3c47389955aeb955804" V1_FALLBACK_VERSION = "v1.2.4" +# The original five-key handoff remains a compatibility contract for the +# Reference/L2 rollout. C2 credential recovery adds versioned keys alongside +# it; it never changes a public key already trusted by an active reader. +LEGACY_KEY_SUBJECTS = { + "stable-trading-system-rs256-v1": "spiffe://qdl/paper/trading-system-stable", + "stable-alpha-binance-rs256-v1": "spiffe://qdl/paper/alpha-binance-stable", + "stable-monitoring-rs256-v1": "spiffe://qdl/paper/monitoring-multivenue-stable", + "stable-alpha-okx-rs256-v1": "spiffe://qdl/paper/alpha-okx-stable", + "stable-reference-l2-rs256-v1": "spiffe://qdl/paper/reference-l2-stable", +} + EXTERNAL_IDENTITY_SPECS = { "stable-monitoring-rs256-v1": { "subject": "spiffe://qdl/paper/monitoring-multivenue-stable", @@ -27,12 +38,32 @@ }, } -ALL_KEY_SUBJECTS = { - "stable-trading-system-rs256-v1": "spiffe://qdl/paper/trading-system-stable", - "stable-alpha-binance-rs256-v1": "spiffe://qdl/paper/alpha-binance-stable", +ALL_KEY_SUBJECTS = dict(LEGACY_KEY_SUBJECTS) + +RECOVERY_IDENTITY_SPECS = { + "stable-monitoring-rs256-v2": { + "subject": "spiffe://qdl/paper/monitoring-multivenue-stable", + "public_key": "monitoring-jwt/public.pem", + }, + "stable-trading-system-rs256-v2": { + "subject": "spiffe://qdl/paper/trading-system-stable", + "public_key": "trading-system-jwt/public.pem", + }, + "stable-alpha-binance-rs256-v2": { + "subject": "spiffe://qdl/paper/alpha-binance-stable", + "public_key": "alpha-binance-jwt/public.pem", + }, + "stable-alpha-okx-rs256-v2": { + "subject": "spiffe://qdl/paper/alpha-okx-stable", + "public_key": "alpha-okx-jwt/public.pem", + }, +} + +RECOVERY_ALL_KEY_SUBJECTS = { + **ALL_KEY_SUBJECTS, **{ key_id: str(value["subject"]) - for key_id, value in EXTERNAL_IDENTITY_SPECS.items() + for key_id, value in RECOVERY_IDENTITY_SPECS.items() }, } @@ -523,14 +554,16 @@ def validate_active_query_environment_commitment( } -def prepare_handoff_environment( +def _prepare_handoff_environment( base_environment: Mapping[str, str], *, extension_dir: str | Path, python_image: str, runtime_binding: Mapping[str, object] | None = None, + identity_specs: Mapping[str, Mapping[str, object]], + approved_key_subjects: Mapping[str, str], ) -> dict[str, str]: - """Append only approved external public signing keys. + """Append only an approved set of external public signing keys. Private key paths deliberately never enter the query/stream environment. """ @@ -570,9 +603,7 @@ def prepare_handoff_environment( for key, value in keys.items() ): raise ValueError("stable JWT public keyring is invalid") - missing_existing = sorted( - {"stable-trading-system-rs256-v1", "stable-alpha-binance-rs256-v1"} - set(keys) - ) + missing_existing = sorted(set(ALL_KEY_SUBJECTS) - set(keys)) if missing_existing: raise ValueError(f"stable JWT public keyring misses existing identities {missing_existing}") prior_subjects_raw = result.get("QDL_STABLE_JWT_KEY_SUBJECTS_JSON") @@ -586,7 +617,7 @@ def prepare_handoff_environment( for key, value in prior_subjects.items() ): raise ValueError("stable JWT key-subject bindings are invalid") - unknown_subjects = sorted(set(prior_subjects) - set(ALL_KEY_SUBJECTS)) + unknown_subjects = sorted(set(prior_subjects) - set(approved_key_subjects)) if unknown_subjects: raise ValueError( "stable JWT key-subject bindings contain unapproved identities " @@ -595,7 +626,7 @@ def prepare_handoff_environment( mismatched_subjects = sorted( key for key, subject in prior_subjects.items() - if ALL_KEY_SUBJECTS[key] != subject + if approved_key_subjects[key] != subject ) if mismatched_subjects: raise ValueError( @@ -603,7 +634,7 @@ def prepare_handoff_environment( f"{mismatched_subjects}" ) extension = Path(extension_dir) - for key_id, spec in EXTERNAL_IDENTITY_SPECS.items(): + for key_id, spec in identity_specs.items(): public_key_path = extension / str(spec["public_key"]) if not public_key_path.is_file(): raise ValueError(f"Phase 10.5-C {key_id} public key is missing") @@ -614,16 +645,50 @@ def prepare_handoff_environment( if prior is not None and prior != public_key: raise ValueError(f"Phase 10.5-C {key_id} conflicts with the existing keyring") keys[key_id] = public_key - if set(keys) != set(ALL_KEY_SUBJECTS): + if set(keys) != set(approved_key_subjects): raise ValueError("Phase 10.5-C JWT keyring does not exactly match approved identities") result["QDL_STABLE_JWT_KEYS_JSON"] = json.dumps(keys, sort_keys=True, separators=(",", ":")) result["QDL_STABLE_JWT_KEY_SUBJECTS_JSON"] = json.dumps( - ALL_KEY_SUBJECTS, sort_keys=True, separators=(",", ":") + dict(approved_key_subjects), sort_keys=True, separators=(",", ":") ) result["QDL_STABLE_PYTHON_IMAGE"] = python_image return result +def prepare_handoff_environment( + base_environment: Mapping[str, str], + *, + extension_dir: str | Path, + python_image: str, + runtime_binding: Mapping[str, object] | None = None, +) -> dict[str, str]: + """Preserve the historical five-key Reference/L2 handoff contract.""" + return _prepare_handoff_environment( + base_environment, + extension_dir=extension_dir, + python_image=python_image, + runtime_binding=runtime_binding, + identity_specs=EXTERNAL_IDENTITY_SPECS, + approved_key_subjects=ALL_KEY_SUBJECTS, + ) + + +def prepare_c2_identity_recovery_environment( + base_environment: Mapping[str, str], + *, + extension_dir: str | Path, + python_image: str, +) -> dict[str, str]: + """Append four rotated C2 keys while retaining every trusted V1 key.""" + return _prepare_handoff_environment( + base_environment, + extension_dir=extension_dir, + python_image=python_image, + identity_specs=RECOVERY_IDENTITY_SPECS, + approved_key_subjects=RECOVERY_ALL_KEY_SUBJECTS, + ) + + def public_handoff_overlay(environment: Mapping[str, str]) -> dict[str, str]: """Select the only two public C2 environment values Compose may layer.""" keys = ("QDL_STABLE_JWT_KEYS_JSON", "QDL_STABLE_JWT_KEY_SUBJECTS_JSON") @@ -682,6 +747,7 @@ def handoff_packet( v1_attestation: Mapping[str, object], runtime_binding: Mapping[str, object] | None = None, query_environment_commitment: Mapping[str, object] | None = None, + approved_key_subjects: Mapping[str, str] = ALL_KEY_SUBJECTS, ) -> dict[str, object]: """Make an auditable packet without serializing a secret-bearing env file.""" if v1_attestation.get("status") != "PASS": @@ -702,7 +768,7 @@ def handoff_packet( raise ValueError(f"Phase 10.5-C handoff environment is missing {missing}") public_keyring = json.loads(environment["QDL_STABLE_JWT_KEYS_JSON"]) subjects = json.loads(environment["QDL_STABLE_JWT_KEY_SUBJECTS_JSON"]) - if set(public_keyring) != set(ALL_KEY_SUBJECTS) or subjects != ALL_KEY_SUBJECTS: + if set(public_keyring) != set(approved_key_subjects) or subjects != dict(approved_key_subjects): raise ValueError("Phase 10.5-C packet identity map is not exact") packet = { "schema": "qdl.phase105c.handoff-packet.v1", diff --git a/qdl/certification/phase105_release.py b/qdl/certification/phase105_release.py index ec76db0a..0f51842c 100644 --- a/qdl/certification/phase105_release.py +++ b/qdl/certification/phase105_release.py @@ -23,6 +23,7 @@ is_explicit_v1_exclusion, requirement_key, ) +from qdl.consumer.release import v2_observation_is_current _SHA256 = re.compile(r"[0-9a-f]{64}\Z") @@ -41,6 +42,10 @@ "cpu_millicores", "rss_bytes", }) +_SESSION_OBSERVATION_FIELDS = frozenset({ + "v2_quality_state", "v2_session_state", "v2_session_liveness_ms", + "v2_complete", "v2_execution_eligible", +}) _RUNTIME_SCHEMA = "qdl.phase105c.runtime-handoff-evidence.v1" _ACCEPTANCE_SCHEMA = "qdl.phase105.v2-identity-acceptance.v1" _FALLBACK_SCHEMA = "qdl.phase105.v1-fallback-return.v1" @@ -142,7 +147,7 @@ def parse_release_observations(raw: object) -> tuple[ReleaseRouteObservation, .. values: list[ReleaseRouteObservation] = [] for index, item in enumerate(raw): value = _mapping(item, f"observation[{index}]") - if set(value) != _OBSERVATION_FIELDS: + if set(value) not in (_OBSERVATION_FIELDS, _OBSERVATION_FIELDS | _SESSION_OBSERVATION_FIELDS): raise ValueError("Phase 10.5-D observation fields differ from public contract") if not all( isinstance(value[field], str) and value[field] @@ -265,19 +270,15 @@ def _measurements_are_current( } for item in observations: product, requirement = requirements[(item.consumer_id, item.requirement_key)] - maximum = requirement.max_freshness_ms if product.route == "V1_PRIMARY": if not is_explicit_v1_exclusion(product, item): return False continue if item.route == "V2_PRIMARY": - ages = (item.v2_source_age_ms, item.v2_receive_age_ms) - if item.v2_gap_open or any(age is None for age in ages): + if not v2_observation_is_current(requirement, item): return False else: return False - if maximum is not None and any(age > maximum for age in ages if age is not None): - return False return True diff --git a/qdl/certification/phase105_release_observations.py b/qdl/certification/phase105_release_observations.py index 0028958f..538af9da 100644 --- a/qdl/certification/phase105_release_observations.py +++ b/qdl/certification/phase105_release_observations.py @@ -27,6 +27,10 @@ OBSERVATION_BUNDLE_SCHEMA = "qdl.phase105.release-observations.v1" MAX_OBSERVATION_AGE_MS = 300_000 _QUALITY_FIELDS = frozenset({"source_age_ms", "receive_age_ms", "gap_open"}) +_SESSION_QUALITY_FIELDS = frozenset({ + "state", "provider_session_state", "provider_session_liveness_ms", + "complete", "execution_eligible", +}) _CAPTURE_FIELDS = frozenset({"captured_at_ms", "cpu_millicores", "rss_bytes"}) _BUNDLE_FIELDS = frozenset({ "schema", @@ -85,13 +89,15 @@ def _require_positive_int(value: object, field: str) -> int: return result -def _quality(value: object, field: str) -> dict[str, int | bool]: - if not isinstance(value, Mapping) or set(value) != _QUALITY_FIELDS: +def _quality(value: object, field: str) -> dict[str, object]: + if not isinstance(value, Mapping) or set(value) not in ( + _QUALITY_FIELDS, _QUALITY_FIELDS | _SESSION_QUALITY_FIELDS + ): raise ValueError(f"Phase 10.5 B3 {field} quality fields are invalid") gap_open = value.get("gap_open") if not isinstance(gap_open, bool): raise ValueError(f"Phase 10.5 B3 {field}.gap_open must be boolean") - return { + result = { "source_age_ms": _require_non_negative_int( value.get("source_age_ms"), f"{field}.source_age_ms" ), @@ -100,12 +106,43 @@ def _quality(value: object, field: str) -> dict[str, int | bool]: ), "gap_open": gap_open, } + if "state" in value: + result.update({key: value[key] for key in _SESSION_QUALITY_FIELDS}) + # Reuse the public observation's strict typed-field validation. + ReleaseRouteObservation( + consumer_id="validate", requirement_key="validate", route="V2_PRIMARY", + reason="validate", v2_source_age_ms=result["source_age_ms"], + v2_receive_age_ms=result["receive_age_ms"], v2_gap_open=gap_open, + v1_source_age_ms=None, v1_receive_age_ms=None, consumer_lag=0, + cpu_millicores=0, rss_bytes=0, **_session_observation_fields(result, result), + ) + return result + + +def _session_observation_fields(primary, secondary) -> dict[str, object]: + if "state" not in primary or "state" not in secondary: + return {} + states = (primary["state"], secondary["state"]) + sessions = (primary["provider_session_state"], secondary["provider_session_state"]) + ages = (primary["provider_session_liveness_ms"], secondary["provider_session_liveness_ms"]) + # Either replica can block readiness; a healthy peer never hides a fault. + return { + "v2_quality_state": "LIVE" if states == ("LIVE", "LIVE") else next(s for s in states if s != "LIVE"), + "v2_session_state": sessions[0] if sessions[0] == sessions[1] else "UNKNOWN", + "v2_session_liveness_ms": max(ages) if all(a is not None for a in ages) else None, + "v2_complete": primary["complete"] and secondary["complete"], + "v2_execution_eligible": primary["execution_eligible"] and secondary["execution_eligible"], + } def _durable_no_cursor_lag(raw: Mapping[str, object], *, index: int) -> int: """Accept a C2-proven live durable handoff with no new cursor event.""" handoff = raw.get("stream_handoff") sessions = raw.get("stream_no_event_sessions") + if handoff == "CURRENT_FINAL_BAR_OBSERVED_NO_CURSOR": + if raw.get("feed") != "BAR" or sessions != ["CURRENT_FINAL_BAR", "CURRENT_FINAL_BAR"]: + raise ValueError("Phase 10.5 B3 quiet final BAR requires both current sessions") + return 0 if handoff not in _DURABLE_NO_CURSOR_HANDOFFS: raise ValueError("Phase 10.5 B3 durable no-cursor handoff is unproven") if ( @@ -119,7 +156,7 @@ def _durable_no_cursor_lag(raw: Mapping[str, object], *, index: int) -> int: return 0 -def compact_view_quality(view: object, *, observed_at_ns: int | None = None) -> dict[str, int | bool]: +def compact_view_quality(view: object, *, observed_at_ns: int | None = None) -> dict[str, object]: """Return age/gap evidence from one already SDK-validated V2 view.""" now_ns = time.time_ns() if observed_at_ns is None else observed_at_ns if isinstance(now_ns, bool) or not isinstance(now_ns, int) or now_ns <= 0: @@ -135,16 +172,21 @@ def compact_view_quality(view: object, *, observed_at_ns: int | None = None) -> raise ValueError("Phase 10.5 B3 view.received_at_ns must be positive") if received_at_ns > now_ns: raise ValueError("Phase 10.5 B3 view.received_at_ns is in the future") - return { + result = { "source_age_ms": source_age_ms, "receive_age_ms": (now_ns - received_at_ns) // 1_000_000, "gap_open": gap_open, } + if all(hasattr(quality, key) for key in _SESSION_QUALITY_FIELDS): + result.update({key: getattr(quality, key) for key in _SESSION_QUALITY_FIELDS}) + return _quality(result, "view") def build_release_observation_bundle( plan: StableReleaseRoutePlan, acceptance: Mapping[str, object], + *, + current_reads: Mapping[str, object] | None = None, ) -> dict[str, object]: """Build exact B3 route evidence from one current C2-compatible receipt.""" if ( @@ -154,14 +196,41 @@ def build_release_observation_bundle( ): raise ValueError("Phase 10.5 B3 acceptance receipt is not the frozen release scope") capture = acceptance.get("release_capture") + current_by_identity = None + if current_reads is not None: + if ( + current_reads.get("schema") != "qdl.release.current-reads.v1" + or current_reads.get("status") != "PASS" + or current_reads.get("release_route_plan_sha256") != plan.digest + or current_reads.get("acceptance_sha256") != _canonical_sha256(acceptance) + or not isinstance(current_reads.get("products"), list) + ): + raise ValueError("Phase 10.5 current reads do not bind to the passing C2") + capture = current_reads.get("release_capture") + current_by_identity = {} + for item in current_reads["products"]: + if not isinstance(item, Mapping): + raise ValueError("Phase 10.5 current read product is invalid") + key = (item.get("consumer_id"), item.get("instrument_uid"), item.get("feed"), + item.get("interval"), item.get("source_policy_id")) + if key in current_by_identity: + raise ValueError("Phase 10.5 current reads duplicate a product") + current_by_identity[key] = item if not isinstance(capture, Mapping) or set(capture) != _CAPTURE_FIELDS: raise ValueError("Phase 10.5 B3 acceptance receipt lacks compact capture metrics") captured_at_ms = _require_positive_int(capture.get("captured_at_ms"), "captured_at_ms") + if current_reads is not None and captured_at_ms < acceptance["release_capture"]["captured_at_ms"]: + raise ValueError("Phase 10.5 current reads predate the passing C2") cpu_millicores = _require_non_negative_int(capture.get("cpu_millicores"), "cpu_millicores") rss_bytes = _require_non_negative_int(capture.get("rss_bytes"), "rss_bytes") raw_products = acceptance.get("products") if not isinstance(raw_products, list): raise ValueError("Phase 10.5 B3 acceptance products are unavailable") + if current_by_identity is not None: + expected_keys = {(item.get("consumer_id"), item.get("instrument_uid"), item.get("feed"), + item.get("interval"), item.get("source_policy_id")) for item in raw_products} + if set(current_by_identity) != expected_keys or len(expected_keys) != len(raw_products): + raise ValueError("Phase 10.5 current reads differ from the complete C2 scope") expected_v2 = { (consumer_id, product.requirement_key): product @@ -174,7 +243,7 @@ def build_release_observation_bundle( for requirement in consumer.manifest.requirements } catalog = StableSourceCatalog.load(plan.source_catalog.path) - measured: dict[tuple[str, str], tuple[dict[str, int | bool], dict[str, int | bool], int]] = {} + measured: dict[tuple[str, str], tuple[dict[str, object], dict[str, object], int]] = {} for index, raw in enumerate(raw_products): if not isinstance(raw, Mapping): raise ValueError(f"Phase 10.5 B3 product[{index}] is invalid") @@ -191,7 +260,14 @@ def build_release_observation_bundle( identity = (consumer_id, route_key) if identity not in expected_v2 or identity in measured: raise ValueError("Phase 10.5 B3 products differ from frozen V2 routes") - quality = raw.get("release_quality") + quality_source = raw.get("closing_v2_read", raw) + if current_by_identity is not None: + quality_source = current_by_identity[(consumer_id, instrument_uid, feed, interval, policy)] + if not isinstance(quality_source, Mapping) or any(quality_source.get(key) != raw.get(key) for key in ( + "consumer_id", "instrument_uid", "feed", "interval", "source_policy_id" + )): + raise ValueError("Phase 10.5 closing quality identity differs from C2") + quality = quality_source.get("release_quality") if not isinstance(quality, Mapping) or set(quality) != {"primary", "secondary"}: raise ValueError(f"Phase 10.5 B3 product[{index}] quality is unavailable") primary = _quality(quality.get("primary"), f"product[{index}].primary") @@ -259,14 +335,18 @@ def build_release_observation_bundle( consumer_lag=consumer_lag, cpu_millicores=cpu_millicores, rss_bytes=rss_bytes, + **_session_observation_fields(primary, secondary), )) - return { + result = { "schema": OBSERVATION_BUNDLE_SCHEMA, "release_route_plan_sha256": plan.digest, "captured_at_ms": captured_at_ms, "acceptance_sha256": _canonical_sha256(acceptance), "observations": [item.public_record() for item in observations], } + if current_reads is not None: + result["current_reads_sha256"] = _canonical_sha256(current_reads) + return result def parse_release_observation_bundle( @@ -277,7 +357,9 @@ def parse_release_observation_bundle( max_age_ms: int = MAX_OBSERVATION_AGE_MS, ) -> tuple[ReleaseRouteObservation, ...]: """Validate a fresh exact B3 observation bundle before certification.""" - if not isinstance(value, Mapping) or set(value) != _BUNDLE_FIELDS: + if not isinstance(value, Mapping) or set(value) not in ( + _BUNDLE_FIELDS, _BUNDLE_FIELDS | {"current_reads_sha256"} + ): raise ValueError("Phase 10.5 B3 observation bundle fields are invalid") if value.get("schema") != OBSERVATION_BUNDLE_SCHEMA: raise ValueError("Phase 10.5 B3 observation bundle schema is invalid") @@ -297,6 +379,10 @@ def parse_release_observation_bundle( or any(character not in "0123456789abcdef" for character in acceptance_sha256) ): raise ValueError("Phase 10.5 B3 observation bundle acceptance digest is invalid") + if "current_reads_sha256" in value: + digest = value["current_reads_sha256"] + if not isinstance(digest, str) or len(digest) != 64 or any(c not in "0123456789abcdef" for c in digest): + raise ValueError("Phase 10.5 B3 current-read digest is invalid") from qdl.certification.phase105_release import parse_release_observations return parse_release_observations(value.get("observations")) diff --git a/qdl/certification/prerequisites.py b/qdl/certification/prerequisites.py index 55c66408..a6f52cb3 100644 --- a/qdl/certification/prerequisites.py +++ b/qdl/certification/prerequisites.py @@ -5,7 +5,7 @@ import re import uuid from dataclasses import dataclass -from enum import StrEnum +from qdl._compat import StrEnum from pathlib import Path from typing import Any diff --git a/qdl/certification/reference_l2_acceptance.py b/qdl/certification/reference_l2_acceptance.py index 40313e03..55e9dd00 100644 --- a/qdl/certification/reference_l2_acceptance.py +++ b/qdl/certification/reference_l2_acceptance.py @@ -48,6 +48,7 @@ _MILLISECOND_NS = 1_000_000 _FUNDING_SETTLEMENT_JITTER_NS = 60_000 * _MILLISECOND_NS _REFERENCE_ACCEPTANCE_BATCH_SIZE = 12 +_STRICT_MARK_FRESHNESS_MS = 2_000 _ACCEPTANCE_TRANSPORT_MARGIN_SECONDS = 15.0 _ACCEPTANCE_TRANSPORT_MAX_SECONDS = 90.0 @@ -157,23 +158,49 @@ def reference_acceptance_batches( """Split the real receipt without changing its complete product scope. Native Binance basis uses one Rust-admitted provider lane, so each request - is intentionally isolated. Every other reference requirement stays in a - bounded batch; no product is dropped or retried through another provider. + is intentionally isolated. A strict mark/index snapshot cannot spend its + entire freshness budget waiting behind other provider reads, so each + product with a declared two-second-or-tighter bound is also isolated. + Every other reference requirement stays in a bounded batch; no product is + dropped or retried through another provider. """ - native_basis = tuple( - product - for product in products - if product.venue == "BINANCE" - and product.requirement.feed is FeedType.BASIS - and product.sdk_requirement.basis_series is BasisSeries.NATIVE + native_basis = tuple(product for product in products if is_rust_admitted_native_basis(product)) + strict_marks = tuple(product for product in products if is_strict_mark_snapshot(product)) + ordinary = tuple( + product for product in products + if not is_rust_admitted_native_basis(product) and not is_strict_mark_snapshot(product) ) - ordinary = tuple(product for product in products if product not in native_basis) chunks = tuple( ordinary[offset:offset + _REFERENCE_ACCEPTANCE_BATCH_SIZE] for offset in range(0, len(ordinary), _REFERENCE_ACCEPTANCE_BATCH_SIZE) ) - return tuple((product,) for product in native_basis) + chunks + return ( + tuple((product,) for product in native_basis) + + tuple((product,) for product in strict_marks) + + chunks + ) + + +def is_rust_admitted_native_basis(product: ReferenceAcceptanceProduct) -> bool: + """Identify the one shared Rust-admitted provider lane exactly once.""" + + return ( + product.venue == "BINANCE" + and product.requirement.feed is FeedType.BASIS + and product.sdk_requirement.basis_series is BasisSeries.NATIVE + ) + + +def is_strict_mark_snapshot(product: ReferenceAcceptanceProduct) -> bool: + """Keep a latency-critical mark/index certificate within its own SLA.""" + + freshness_ms = product.requirement.max_freshness_ms + return ( + product.requirement.feed is FeedType.MARK_INDEX_PRICE + and freshness_ms is not None + and freshness_ms <= _STRICT_MARK_FRESHNESS_MS + ) def acceptance_transport_timeout_seconds(provider_deadline_seconds: float) -> float: @@ -374,7 +401,16 @@ def reference_evidence( or item.problem is not None or item.data is None ): - raise ValueError("reference response identity or availability differs from demand") + actual_product = getattr(getattr(item, "product", None), "value", None) + problem_code = getattr(getattr(item, "problem", None), "code", None) + problem_detail = getattr(getattr(item, "problem", None), "detail", None) + raise ValueError( + "reference response identity or availability differs from demand " + f"expected_uid={product.instrument_uid} expected_product={request.product.value} " + f"actual_uid={getattr(item, 'instrument_uid', None)} " + f"actual_product={actual_product} status={getattr(item, 'status', None)} " + f"problem_code={problem_code} problem_detail={problem_detail}" + ) data = item.data if data.instrument_uid != product.instrument_uid or data.product.value != request.product.value: raise ValueError("reference payload identity differs from demand") diff --git a/qdl/certification/release.py b/qdl/certification/release.py index 280c1768..2c00338c 100644 --- a/qdl/certification/release.py +++ b/qdl/certification/release.py @@ -5,7 +5,10 @@ import re import subprocess import time -import tomllib +try: # Python 3.11+ + import tomllib +except ModuleNotFoundError: # Python 3.10 support is declared by pyproject. + import tomli as tomllib from pathlib import Path from typing import Any diff --git a/qdl/consumer/manifest.py b/qdl/consumer/manifest.py index 83788192..e984fef3 100644 --- a/qdl/consumer/manifest.py +++ b/qdl/consumer/manifest.py @@ -4,7 +4,7 @@ import json import time from dataclasses import dataclass -from enum import StrEnum +from qdl._compat import StrEnum from pathlib import Path from typing import Any @@ -177,8 +177,8 @@ def from_mapping(payload: Any) -> ConsumerManifest: }: raise ValueError("consumer manifest spec contains unknown fields") requirements = spec.get("requirements") - if not isinstance(requirements, list) or not 1 <= len(requirements) <= 100: - raise ValueError("consumer manifest requires 1..100 requirements") + if not isinstance(requirements, list) or not 1 <= len(requirements) <= 256: + raise ValueError("consumer manifest requires 1..256 requirements") permissions = spec.get("permissions") purposes = spec.get("purposes") quotas = spec.get("quotas") diff --git a/qdl/consumer/realtime_route.py b/qdl/consumer/realtime_route.py index 005e9ca2..abf80c16 100644 --- a/qdl/consumer/realtime_route.py +++ b/qdl/consumer/realtime_route.py @@ -9,7 +9,7 @@ from __future__ import annotations from dataclasses import dataclass -from enum import StrEnum +from qdl._compat import StrEnum from qdl.consumer.manifest import ConsumerManifest, ConsumerRoute from qdl.query import ( diff --git a/qdl/consumer/release.py b/qdl/consumer/release.py index efc231c2..8304489c 100644 --- a/qdl/consumer/release.py +++ b/qdl/consumer/release.py @@ -552,6 +552,11 @@ class ReleaseRouteObservation: consumer_lag: int cpu_millicores: int rss_bytes: int + v2_quality_state: str | None = None + v2_session_state: str | None = None + v2_session_liveness_ms: int | None = None + v2_complete: bool | None = None + v2_execution_eligible: bool | None = None def __post_init__(self) -> None: if ( @@ -574,9 +579,24 @@ def __post_init__(self) -> None: for value in (self.consumer_lag, self.cpu_millicores, self.rss_bytes) ): raise ValueError("release route resource metrics must be non-negative") + typed = (self.v2_quality_state, self.v2_session_state, + self.v2_session_liveness_ms, self.v2_complete, self.v2_execution_eligible) + if any(value is not None for value in typed): + if ( + self.v2_quality_state not in {"LIVE", "STALE", "GAPPED", "OFFLINE", "UNAVAILABLE", "MARKET_CLOSED"} + or self.v2_session_state not in {"LIVE", "STALE", "DISCONNECTED", "UNKNOWN", "NOT_APPLICABLE"} + or not isinstance(self.v2_complete, bool) + or not isinstance(self.v2_execution_eligible, bool) + or (self.v2_session_liveness_ms is not None and ( + isinstance(self.v2_session_liveness_ms, bool) + or not isinstance(self.v2_session_liveness_ms, int) + or self.v2_session_liveness_ms < 0 + )) + ): + raise ValueError("release route typed session evidence is invalid") def public_record(self) -> dict[str, object]: - return { + result = { "consumer_id": self.consumer_id, "requirement_key": self.requirement_key, "route": self.route, @@ -590,6 +610,44 @@ def public_record(self) -> dict[str, object]: "cpu_millicores": self.cpu_millicores, "rss_bytes": self.rss_bytes, } + if self.v2_quality_state is not None: + result.update({ + "v2_quality_state": self.v2_quality_state, + "v2_session_state": self.v2_session_state, + "v2_session_liveness_ms": self.v2_session_liveness_ms, + "v2_complete": self.v2_complete, + "v2_execution_eligible": self.v2_execution_eligible, + }) + return result + + +def v2_observation_is_current(requirement, observed: ReleaseRouteObservation) -> bool: + """Distinguish current executable prices from a verified quiet event feed.""" + ages = (observed.v2_source_age_ms, observed.v2_receive_age_ms) + if observed.v2_gap_open or any(age is None for age in ages): + return False + session_live = ( + observed.v2_quality_state == "LIVE" and observed.v2_complete is True + and observed.v2_session_state == "LIVE" + and observed.v2_session_liveness_ms is not None + and requirement.max_session_liveness_ms is not None + and observed.v2_session_liveness_ms <= requirement.max_session_liveness_ms + ) + if observed.v2_quality_state is not None: + if observed.v2_quality_state != "LIVE" or observed.v2_complete is not True: + return False + if observed.v2_session_state not in {"LIVE", "NOT_APPLICABLE"}: + return False + if requirement.max_session_liveness_ms is not None and not session_live: + return False + maximum = requirement.max_freshness_ms + if maximum is None or all(age <= maximum for age in ages): + return True + return ( + requirement.feed.value in {"TRADE", "BOOK_DELTA"} + and requirement.effective_event_recency_policy.value == "OBSERVE" + and session_live and observed.v2_execution_eligible is False + ) @dataclass(frozen=True, slots=True) @@ -681,14 +739,7 @@ def evaluate_release_readiness( v1_primary_count += 1 continue if observed.route == _V2_ROUTE: - if ( - observed.v2_gap_open - or observed.v2_source_age_ms is None - or ( - requirement.max_freshness_ms is not None - and observed.v2_source_age_ms > requirement.max_freshness_ms - ) - ): + if not v2_observation_is_current(requirement, observed): blocked_count += 1 else: v2_primary_count += 1 diff --git a/qdl/consumer/universal_release.py b/qdl/consumer/universal_release.py index 432b46eb..3ca23355 100644 --- a/qdl/consumer/universal_release.py +++ b/qdl/consumer/universal_release.py @@ -11,7 +11,7 @@ from collections import Counter from dataclasses import dataclass -from enum import StrEnum +from qdl._compat import StrEnum import hashlib import json from pathlib import Path @@ -34,7 +34,8 @@ _SCHEMA = "qdl.v2.universal-release-manifest.v1" _POLICY_SCHEMA = "qdl.v2.universal-release-policy.v1" _COVERAGE_SCHEMA = "qdl.v2.universal-release-coverage.v1" -_CONSUMER_ROUTE_BINDING_SCHEMA = "qdl.v2.consumer-route-binding.v1" +_CONSUMER_ROUTE_BINDING_SCHEMA_V1 = "qdl.v2.consumer-route-binding.v1" +_CONSUMER_ROUTE_BINDING_SCHEMA_V2 = "qdl.v2.consumer-route-binding.v2" _V2_VENUES = frozenset({"BINANCE", "OKX"}) _REALTIME_FEEDS = frozenset({DemandFeed.TRADE, DemandFeed.QUOTE, DemandFeed.BAR}) _BOOK_FEEDS = frozenset({DemandFeed.BOOK_SNAPSHOT, DemandFeed.BOOK_DELTA}) @@ -844,6 +845,10 @@ class ConsumerRouteBinding: independent_v1_venues: tuple[str, ...] products: tuple[UniversalReleaseProduct, ...] binding_sha256: str | None = None + # V1 bindings remain portable for existing unpromoted consumers. A V2 + # binding seals the server-side consumer-manifest revision as well, so a + # JWT minted for another revision fails before any data-plane connection. + consumer_manifest_revision: int | None = None def __post_init__(self) -> None: object.__setattr__(self, "consumer_id", _text(self.consumer_id, "binding consumer_id")) @@ -852,6 +857,11 @@ def __post_init__(self) -> None: self.capability_matrix_revision, "consumer route binding capability matrix revision", ) + if self.consumer_manifest_revision is not None: + _positive_int( + self.consumer_manifest_revision, + "consumer route binding consumer manifest revision", + ) for field in ( "universal_manifest_sha256", "policy_sha256", "capability_matrix_sha256", "inventory_sha256", @@ -882,8 +892,12 @@ def __post_init__(self) -> None: object.__setattr__(self, "binding_sha256", expected) def _canonical_without_digest(self) -> dict[str, object]: - return { - "schema": _CONSUMER_ROUTE_BINDING_SCHEMA, + canonical = { + "schema": ( + _CONSUMER_ROUTE_BINDING_SCHEMA_V2 + if self.consumer_manifest_revision is not None + else _CONSUMER_ROUTE_BINDING_SCHEMA_V1 + ), "contract_version": "2.0.0", "consumer_id": self.consumer_id, "consumer_class": self.consumer_class.value, @@ -899,6 +913,9 @@ def _canonical_without_digest(self) -> dict[str, object]: "independent_v1_venues": list(self.independent_v1_venues), "products": [item.canonical_mapping() for item in self.products], } + if self.consumer_manifest_revision is not None: + canonical["consumer_manifest_revision"] = self.consumer_manifest_revision + return canonical def canonical_mapping(self) -> dict[str, object]: return {**self._canonical_without_digest(), "binding_sha256": self.binding_sha256} @@ -910,6 +927,7 @@ def from_manifest( *, consumer_id: str, independent_v1_venues: tuple[str, ...] = ("DNSE",), + consumer_manifest_revision: int | None = None, ) -> "ConsumerRouteBinding": selected = tuple( item for item in manifest.products if item.consumer_id == _text(consumer_id, "binding consumer_id") @@ -931,17 +949,31 @@ def from_manifest( v1_rollback=manifest.v1_rollback, independent_v1_venues=independent_v1_venues, products=tuple(sorted(selected, key=lambda item: item.requirement_id)), + consumer_manifest_revision=consumer_manifest_revision, ) @classmethod def from_canonical_mapping(cls, value: Mapping[str, object]) -> "ConsumerRouteBinding": - expected = { + base_expected = { "schema", "contract_version", "consumer_id", "consumer_class", "release_revision", "universal_manifest_sha256", "policy_sha256", "capability_matrix", "inventory_sha256", "v1_rollback", "independent_v1_venues", "products", "binding_sha256", } - if set(value) != expected or value.get("schema") != _CONSUMER_ROUTE_BINDING_SCHEMA: + schema = value.get("schema") + if schema == _CONSUMER_ROUTE_BINDING_SCHEMA_V1: + expected = base_expected + consumer_manifest_revision = None + elif schema == _CONSUMER_ROUTE_BINDING_SCHEMA_V2: + expected = base_expected | {"consumer_manifest_revision"} + consumer_manifest_revision = _positive_int( + value.get("consumer_manifest_revision"), + "consumer route binding consumer manifest revision", + ) + else: + expected = set() + consumer_manifest_revision = None + if set(value) != expected: raise ValueError("consumer route binding schema or fields are invalid") if value.get("contract_version") != "2.0.0": raise ValueError("consumer route binding contract version is invalid") @@ -1027,6 +1059,7 @@ def from_canonical_mapping(cls, value: Mapping[str, object]) -> "ConsumerRouteBi independent_v1_venues=tuple(str(item) for item in venues), products=tuple(products), binding_sha256=str(value["binding_sha256"]), + consumer_manifest_revision=consumer_manifest_revision, ) diff --git a/qdl/data_quality/engine.py b/qdl/data_quality/engine.py index 4ced9817..e4d73e27 100644 --- a/qdl/data_quality/engine.py +++ b/qdl/data_quality/engine.py @@ -2,7 +2,9 @@ from collections import deque from dataclasses import dataclass, field -from enum import IntEnum, StrEnum +from enum import IntEnum + +from qdl._compat import StrEnum class ValidationLevel(IntEnum): diff --git a/qdl/data_quality/source_authority.py b/qdl/data_quality/source_authority.py index 339a08c4..0e189266 100644 --- a/qdl/data_quality/source_authority.py +++ b/qdl/data_quality/source_authority.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from enum import StrEnum +from qdl._compat import StrEnum class SourceRole(StrEnum): diff --git a/qdl/domain/lifecycle.py b/qdl/domain/lifecycle.py index 318f1898..880c5ada 100644 --- a/qdl/domain/lifecycle.py +++ b/qdl/domain/lifecycle.py @@ -1,6 +1,6 @@ from __future__ import annotations -from enum import StrEnum +from qdl._compat import StrEnum class BarLifecycle(StrEnum): diff --git a/qdl/domain/quantity.py b/qdl/domain/quantity.py index 6084eaf6..10fe88b8 100644 --- a/qdl/domain/quantity.py +++ b/qdl/domain/quantity.py @@ -1,6 +1,6 @@ from __future__ import annotations -from enum import StrEnum +from qdl._compat import StrEnum from qdl.common.v1 import common_pb2 diff --git a/qdl/query/contracts.py b/qdl/query/contracts.py index 72a40639..8a2ecc79 100644 --- a/qdl/query/contracts.py +++ b/qdl/query/contracts.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from enum import StrEnum +from qdl._compat import StrEnum from qdl.warmup.contracts import WarmupSpecification diff --git a/qdl/query/entitlement.py b/qdl/query/entitlement.py index efc42f44..f6ec182a 100644 --- a/qdl/query/entitlement.py +++ b/qdl/query/entitlement.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from enum import StrEnum +from qdl._compat import StrEnum class AccessPurpose(StrEnum): diff --git a/qdl/query/lifecycle.py b/qdl/query/lifecycle.py index 93350692..c456c76f 100644 --- a/qdl/query/lifecycle.py +++ b/qdl/query/lifecycle.py @@ -1,6 +1,6 @@ from __future__ import annotations -from enum import StrEnum +from qdl._compat import StrEnum from qdl.domain.lifecycle import BarLifecycle from qdl.query.contracts import FeedType diff --git a/qdl/query/service.py b/qdl/query/service.py index ae56b4b7..ee06f74c 100644 --- a/qdl/query/service.py +++ b/qdl/query/service.py @@ -41,6 +41,7 @@ from qdl.reference.batch import ReferenceBatch from qdl.reference.contracts import ( ReferenceBatchResult, + ReferenceProduct, ReferenceRequest, ReferenceStatus, ) @@ -338,6 +339,9 @@ async def work(requirement: DataRequirement) -> WarmupResult: raise def provider(requirement: DataRequirement) -> str: + local = getattr(self.backend, "warmup_is_local", None) + if callable(local) and local(requirement): + return "LOCAL_CANONICAL_CACHE" try: return self.instruments.get(requirement.instrument_uid).identity.venue except KeyError: @@ -518,10 +522,12 @@ async def reference_data_batch_async( admitted.append((index, requirement, request)) async def work( - candidate: tuple[int, ReferenceDataRequirement, ReferenceRequest] + candidate: tuple[int, ReferenceDataRequirement, ReferenceRequest], + *, + bypass_cache: bool = False, ) -> ReferenceBatchResult: _index, _requirement, request = candidate - result = await self.reference_batch.fetch_one(request) + result = await self.reference_batch.fetch_one(request, bypass_cache=bypass_cache) # Rust provider admission deliberately communicates bounded # pressure through a typed retry delay. Keep Rust as the only # admission authority and let the shared executor honor that @@ -543,7 +549,49 @@ async def work( provider=lambda candidate: candidate[2].instrument.identity.venue, deadline_ms=lambda candidate: candidate[1].deadline_ms, ) + + # Revalidate after every bounded initial task has returned. A current + # mark/index snapshot can become stale while another item in the same + # batch finishes. Refresh those exact current-at-receipt snapshots + # once here, immediately before response assembly. A transient stale + # provider MARK/INDEX row gets the same one bounded, cache-bypassing + # re-read; any still-stale result remains fail-closed. + refresh_candidates = [] for execution in executions: + if execution.error is not None or execution.value is None: + continue + _index, requirement, request = execution.item + problem = self._reference_problem(requirement, request, execution.value) + if ( + problem is not None + and problem.code is CanonicalErrorCode.DATA_STALE + and self._reference_snapshot_requires_refresh( + requirement, + request, + execution.value, + ) + ): + refresh_candidates.append(execution.item) + refresh_by_index = {candidate[0]: candidate for candidate in refresh_candidates} + + for initial_execution in executions: + execution = initial_execution + refresh_candidate = refresh_by_index.get(initial_execution.item[0]) + if refresh_candidate is not None: + # A bounded batch refresh can itself make an early MARK/INDEX + # result stale before response assembly. Re-read and validate + # this exact already-admitted item at its assembly turn instead. + # It is still one cache-bypass recovery through the same + # provider lane; a second stale result remains fail-closed. + execution = ( + await self.warmup_executor.execute( + (refresh_candidate,), + work=lambda candidate: work(candidate, bypass_cache=True), + identity=lambda candidate: candidate[2].cache_key, + provider=lambda candidate: candidate[2].instrument.identity.venue, + deadline_ms=lambda candidate: candidate[1].deadline_ms, + ) + )[0] index, requirement, request = execution.item if execution.error is not None: retry_after_ms = getattr(execution.error, "retry_after_ms", None) @@ -590,6 +638,64 @@ async def work( } return ReferenceBatchQueryResult(request_id, resolved) + def _reference_snapshot_requires_refresh( + self, + requirement: ReferenceDataRequirement, + request: ReferenceRequest, + result: ReferenceBatchResult, + ) -> bool: + """Allow one recovery read for a freshness-governed mark/index snapshot. + + A batch can age an otherwise current cache entry while unrelated work + completes. Separately, exchanges can expose a briefly delayed mark or + index timestamp even though the same declared provider lane is current + on the immediate next read. Both cases get one cache-bypassing re-read + through the same bounded admission path. This deliberately excludes + history and every other reference product: no freshness bound is + relaxed and a second stale observation remains terminal. + """ + + if self._reference_snapshot_was_current_at_receipt( + requirement, + request, + result, + ): + return True + return ( + requirement.max_freshness_ms is not None + and not request.is_history + and request.product is ReferenceProduct.MARK_INDEX_PRICE + and result.status is ReferenceStatus.OK + ) + + def _reference_snapshot_was_current_at_receipt( + self, + requirement: ReferenceDataRequirement, + request: ReferenceRequest, + result: ReferenceBatchResult, + ) -> bool: + """Identify an internally aged current snapshot without masking source staleness.""" + + freshness_ms = requirement.max_freshness_ms + if ( + freshness_ms is None + or request.is_history + or result.status is not ReferenceStatus.OK + or result.received_at_ns <= 0 + ): + return False + newest_observed_ns = max( + (item.observed_at_ns for item in result.observations), + default=0, + ) + if newest_observed_ns <= 0: + return False + source_age_at_receipt_ms = max( + 0, + (result.received_at_ns - newest_observed_ns) // 1_000_000, + ) + return source_age_at_receipt_ms <= freshness_ms + def _reference_problem( self, requirement: ReferenceDataRequirement, diff --git a/qdl/reference/batch.py b/qdl/reference/batch.py index 4f6f809f..ca5cc3e9 100644 --- a/qdl/reference/batch.py +++ b/qdl/reference/batch.py @@ -154,12 +154,28 @@ async def fetch(self, requests: tuple[ReferenceRequest, ...]) -> tuple[Reference return tuple(await asyncio.gather(*(self.fetch_one(request) for request in requests))) - async def fetch_one(self, request: ReferenceRequest) -> ReferenceBatchResult: + async def fetch_one( + self, + request: ReferenceRequest, + *, + bypass_cache: bool = False, + ) -> ReferenceBatchResult: + """Resolve one request, optionally replacing a stale cached snapshot. + + ``bypass_cache`` is intentionally an internal query-service recovery + primitive. It preserves the request identity and provider lane while + preventing an already diagnosed stale cache entry from being returned + a second time. It is not a generic retry policy. + """ key = request.cache_key now = self._monotonic() async with self._lock: self._evict_expired(now) - cached = self._cache.get(key) + if bypass_cache: + self._cache.pop(key, None) + cached = None + else: + cached = self._cache.get(key) if cached is not None: self._cache.move_to_end(key) self._cache_hits += 1 @@ -191,7 +207,7 @@ async def _fetch_uncached( self._inflight.pop(key, None) async def _resolve_request(self, request: ReferenceRequest) -> ReferenceBatchResult: - received_at_ns = self._clock_ns() + request_started_at_ns = self._clock_ns() try: profile = self._capability_resolver(request.instrument) capability_name = product_feed_name(request.product) @@ -200,7 +216,7 @@ async def _resolve_request(self, request: ReferenceRequest) -> ReferenceBatchRes return self._unavailable_result( request, FeedCapability(CapabilityAvailability.UNAVAILABLE, constraint=str(error)), - received_at_ns, + request_started_at_ns, "ADAPTER_UNAVAILABLE", str(error), ) @@ -208,7 +224,7 @@ async def _resolve_request(self, request: ReferenceRequest) -> ReferenceBatchRes return self._unavailable_result( request, FeedCapability(CapabilityAvailability.UNAVAILABLE, constraint=str(error)), - received_at_ns, + request_started_at_ns, "CAPABILITY_UNDECLARED", str(error), ) @@ -217,7 +233,7 @@ async def _resolve_request(self, request: ReferenceRequest) -> ReferenceBatchRes return self._unavailable_result( request, capability, - received_at_ns, + request_started_at_ns, "CAPABILITY_UNAVAILABLE", capability.constraint or "provider capability is unavailable", ) @@ -227,7 +243,7 @@ async def _resolve_request(self, request: ReferenceRequest) -> ReferenceBatchRes return self._unavailable_result( request, capability, - received_at_ns, + request_started_at_ns, "ADAPTER_UNAVAILABLE", f"no registered reference adapter for {request.provider_key[0]}/{request.provider_key[1]}", ) @@ -241,7 +257,9 @@ async def _resolve_request(self, request: ReferenceRequest) -> ReferenceBatchRes self._source_calls += 1 fetched = await asyncio.wait_for( adapter.fetch( - request, capability=capability, received_at_ns=received_at_ns + request, + capability=capability, + received_at_ns=request_started_at_ns, ), timeout=self._policy.request_timeout_seconds, ) @@ -250,7 +268,7 @@ async def _resolve_request(self, request: ReferenceRequest) -> ReferenceBatchRes return self._error_result( request, capability, - received_at_ns, + request_started_at_ns, "PROVIDER_TIMEOUT", "reference provider request exceeded its bounded deadline", ) @@ -258,7 +276,7 @@ async def _resolve_request(self, request: ReferenceRequest) -> ReferenceBatchRes return self._unavailable_result( request, capability, - received_at_ns, + request_started_at_ns, "CAPABILITY_UNAVAILABLE", str(error), ) @@ -266,20 +284,29 @@ async def _resolve_request(self, request: ReferenceRequest) -> ReferenceBatchRes return self._error_result( request, capability, - received_at_ns, + request_started_at_ns, "PROVIDER_RETRY_EXHAUSTED", str(error), retry_after_ms=getattr(error, "retry_after_ms", None), ) except ReferenceProviderError as error: return self._error_result( - request, capability, received_at_ns, "PROVIDER_PROTOCOL", str(error) + request, + capability, + request_started_at_ns, + "PROVIDER_PROTOCOL", + str(error), ) except Exception as error: # adapter failures are isolated to this item return self._error_result( - request, capability, received_at_ns, "PROVIDER_FAILURE", type(error).__name__ + request, + capability, + request_started_at_ns, + "PROVIDER_FAILURE", + type(error).__name__, ) + received_at_ns = self._clock_ns() if not fetched.observations: return ReferenceBatchResult( request=request, diff --git a/qdl/runtime/execution_l2.py b/qdl/runtime/execution_l2.py new file mode 100644 index 00000000..71fff3ce --- /dev/null +++ b/qdl/runtime/execution_l2.py @@ -0,0 +1,196 @@ +"""Manifest-derived execution-grade L2 materialization policy. + +The acquisition file owns provider protocol and bootstrap semantics. The +production demand document owns which physical books are actually execution +products. This module joins the two contracts and prevents a hand-maintained +symbol allow-list from silently drifting after demand expands. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +import yaml + +from qdl.query import FeedType +from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_deployment import StableAcquisitionPlan + + +EXECUTION_L2_MATERIALIZATION_INTERVAL_MS = 1_000 +_BOOK_FEEDS = frozenset({FeedType.BOOK_SNAPSHOT, FeedType.BOOK_DELTA}) + + +@dataclass(frozen=True, slots=True) +class ExecutionL2MaterializationPlan: + """One explicit execution-book set compiled from production demand.""" + + demand_path: Path + demand_sha256: str + source_ids: tuple[str, ...] + binding_ids: tuple[str, ...] + materialized_snapshot_interval_ms: int + + def evidence(self) -> dict[str, object]: + return { + "demand_path": str(self.demand_path), + "demand_sha256": self.demand_sha256, + "source_ids": list(self.source_ids), + "binding_ids": list(self.binding_ids), + "materialized_snapshot_interval_ms": self.materialized_snapshot_interval_ms, + } + + +def _required_text(value: object, field: str) -> str: + result = str(value or "").strip() + if not result: + raise ValueError(f"execution L2 demand {field} is required") + return result + + +def _load_execution_book_rows(path: Path) -> tuple[dict[str, str], ...]: + try: + encoded = path.read_bytes() + payload = yaml.safe_load(encoded) + except (OSError, UnicodeDecodeError, yaml.YAMLError) as error: + raise ValueError(f"execution L2 demand is unreadable: {path}") from error + if not isinstance(payload, Mapping) or payload.get("schema") != "qdl.v2.production-demand.v1": + raise ValueError("execution L2 demand schema is invalid") + consumers = payload.get("consumers") + if not isinstance(consumers, list) or not consumers: + raise ValueError("execution L2 demand has no consumers") + rows: list[dict[str, str]] = [] + for consumer in consumers: + if not isinstance(consumer, Mapping): + raise ValueError("execution L2 demand consumer is invalid") + if str(consumer.get("consumer_grade") or "").strip().upper() != "EXECUTION": + continue + requirements = consumer.get("requirements") + if not isinstance(requirements, list): + raise ValueError("execution L2 demand requirements are invalid") + for requirement in requirements: + if not isinstance(requirement, Mapping): + raise ValueError("execution L2 demand requirement is invalid") + try: + feed = FeedType(_required_text(requirement.get("feed"), "feed").upper()) + except ValueError as error: + raise ValueError("execution L2 demand feed is invalid") from error + if feed not in _BOOK_FEEDS: + continue + depth = requirement.get("depth_per_side") + freshness = requirement.get("max_freshness_ms") + if ( + isinstance(depth, bool) + or not isinstance(depth, int) + or depth < 1 + or isinstance(freshness, bool) + or not isinstance(freshness, int) + or freshness < 1 + or requirement.get("require_live") is not True + ): + raise ValueError("execution L2 demand acquisition fields are invalid") + rows.append({ + "venue": _required_text(requirement.get("venue"), "venue").upper(), + "market": _required_text(requirement.get("market"), "market").upper(), + "product_type": _required_text(requirement.get("product_type"), "product_type").upper(), + "native_symbol": _required_text(requirement.get("native_symbol"), "native_symbol").upper(), + "source_policy_id": _required_text( + requirement.get("source_policy_id"), "source_policy_id" + ), + "feed": feed.value, + }) + if not rows: + raise ValueError("execution L2 demand has no execution-grade book pairs") + return tuple(rows) + + +def execution_l2_materialization_plan( + *, + demand_path: str | Path, + catalog: StableSourceCatalog, + acquisition: StableAcquisitionPlan, + materialized_snapshot_interval_ms: int = EXECUTION_L2_MATERIALIZATION_INTERVAL_MS, +) -> ExecutionL2MaterializationPlan: + """Resolve the only L2 sources eligible for hot execution materialization.""" + + if not 100 <= materialized_snapshot_interval_ms <= 60_000: + raise ValueError("execution L2 materialization cadence is outside bounds") + path = Path(demand_path).resolve() + encoded = path.read_bytes() + rows = _load_execution_book_rows(path) + catalog_by_identity = { + ( + binding.instrument.identity.venue, + binding.instrument.identity.market, + binding.instrument.identity.product_type.value, + binding.instrument.native_symbol.upper(), + binding.source_policy_id, + binding.feed.value, + ): binding + for binding in catalog.bindings + } + acquisition_by_id = {binding.binding_id: binding for binding in acquisition.bindings} + source_rows: dict[str, dict[str, object]] = {} + for row in rows: + key = ( + row["venue"], + row["market"], + row["product_type"], + row["native_symbol"], + row["source_policy_id"], + row["feed"], + ) + source = catalog_by_identity.get(key) + if source is None: + raise ValueError( + "execution L2 demand has no catalog binding: " + "/".join(key) + ) + acquired = acquisition_by_id.get(source.binding_id) + if acquired is None or acquired.l2 is None: + raise ValueError( + f"execution L2 demand is not backed by an acquired L2 source: {source.binding_id}" + ) + record = source_rows.setdefault( + source.source_id, + { + "feeds": set(), + "binding_ids": set(), + "l2": acquired.l2, + }, + ) + if record["l2"] != acquired.l2: + raise ValueError(f"execution L2 aliases disagree: {source.source_id}") + feeds = record["feeds"] + bindings = record["binding_ids"] + assert isinstance(feeds, set) and isinstance(bindings, set) + feeds.add(source.feed) + bindings.add(source.binding_id) + + source_ids: list[str] = [] + binding_ids: list[str] = [] + for source_id, record in sorted(source_rows.items()): + feeds = record["feeds"] + bindings = record["binding_ids"] + l2 = record["l2"] + assert isinstance(feeds, set) and isinstance(bindings, set) + if feeds != _BOOK_FEEDS or len(bindings) != 2: + raise ValueError(f"execution L2 source lacks a complete snapshot/delta pair: {source_id}") + if l2.snapshot_refresh_seconds != 30: + raise ValueError(f"execution L2 provider refresh must remain 30 seconds: {source_id}") + if l2.materialized_snapshot_interval_ms != materialized_snapshot_interval_ms: + raise ValueError( + "execution L2 materialization differs from the common policy: " + f"{source_id}" + ) + source_ids.append(source_id) + binding_ids.extend(sorted(bindings)) + return ExecutionL2MaterializationPlan( + demand_path=path, + demand_sha256=hashlib.sha256(encoded).hexdigest(), + source_ids=tuple(source_ids), + binding_ids=tuple(sorted(binding_ids)), + materialized_snapshot_interval_ms=materialized_snapshot_interval_ms, + ) diff --git a/qdl/runtime/routed_query.py b/qdl/runtime/routed_query.py index 9d645a11..f6837fea 100644 --- a/qdl/runtime/routed_query.py +++ b/qdl/runtime/routed_query.py @@ -60,6 +60,15 @@ def routes_to_pass_through(self, requirement: DataRequirement) -> bool: return False return self.pass_through.serves(requirement) + def warmup_is_local(self, requirement: DataRequirement) -> bool: + # A fresh-snapshot recovery may call a provider after a cache miss. + # It must keep provider admission even when the current cache is full. + return self._binding_exists(requirement) and not ( + requirement.recovery is RecoveryPolicy.FRESH_SNAPSHOT + and self.pass_through is not None + and self.pass_through.serves(requirement) + ) + def history(self, requirement: DataRequirement) -> HistoryResult | None: if self._binding_exists(requirement): result = self.spool.history(requirement) diff --git a/qdl/runtime/stable.py b/qdl/runtime/stable.py index 9e825d9f..3d4f9801 100644 --- a/qdl/runtime/stable.py +++ b/qdl/runtime/stable.py @@ -31,6 +31,7 @@ MeasuredRuntimeReadiness, ) from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_capacity import STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW from qdl.runtime.stable_deployment import validate_shared_authority_record from qdl.runtime.stable_ingest import ( StableHttpCanonicalSink, @@ -62,6 +63,40 @@ logger = logging.getLogger(__name__) +_STABLE_SPOOL_RECORD_FLOOR = 1_000_000 + + +@dataclass(frozen=True, slots=True) +class StableSpoolCapacity: + """Catalog-derived hard bounds for the shared canonical replay cache.""" + + physical_partitions: int + max_partition_records: int + max_records: int + + +def stable_spool_capacity(catalog: StableSourceCatalog) -> StableSpoolCapacity: + """Keep global capacity compatible with the catalog's physical windows. + + Snapshot and delta bindings for a single book deliberately share one + partition. The record bound must therefore count physical keys rather than + logical product requirements, otherwise a larger manifest can deadlock the + shared cache before any declared partition window is reached. + """ + + physical_partitions = len({binding.partition_key for binding in catalog.bindings}) + if physical_partitions <= 0: + raise ValueError("stable spool requires at least one physical partition") + return StableSpoolCapacity( + physical_partitions=physical_partitions, + max_partition_records=STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, + max_records=max( + _STABLE_SPOOL_RECORD_FLOOR, + physical_partitions * STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, + ), + ) + + def _env_flag( env: Mapping[str, str], name: str, *, default: bool ) -> bool: @@ -156,6 +191,8 @@ class StableRuntimeConfig: stream_ingest_urls: tuple[str, ...] = () max_pending_records: int = 10_000 max_pending_bytes: int = 256 * 1024 * 1024 + projector_max_batch_records: int = 128 + projector_max_batch_bytes: int = 8 * 1024 * 1024 # Off unless a deployment turns it on. Declaring catalog metadata for an # instrument must never open the pass-through product by itself. pass_through_enabled: bool = False @@ -222,6 +259,13 @@ def __post_init__(self) -> None: raise ValueError("stable runtime ports/bounds must be positive") if self.max_buffer_events > 10_000 or self.max_replay_events > 10_000: raise ValueError("stable stream/replay bounds exceed contract maximum") + if self.role == "projector_v2": + if not 1 <= self.projector_max_batch_records <= 512: + raise ValueError("stable projector batch bound must be 1..512") + if self.max_pending_records < self.projector_max_batch_records: + raise ValueError("stable projector pending records must cover one batch") + if not 1 <= self.projector_max_batch_bytes <= self.max_pending_bytes: + raise ValueError("stable projector batch byte bound is invalid") if not 5 <= self.lease_ttl_seconds <= 300 or not ( 0 < self.lease_renew_seconds < self.lease_ttl_seconds ): @@ -318,6 +362,12 @@ def from_environment( stream_ingest_urls=tuple(str(value) for value in urls_raw), max_pending_records=int(env.get("QDL_STABLE_MAX_PENDING_RECORDS", "10000")), max_pending_bytes=int(env.get("QDL_STABLE_MAX_PENDING_BYTES", "268435456")), + projector_max_batch_records=int( + env.get("QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS", "128") + ), + projector_max_batch_bytes=int( + env.get("QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES", "8388608") + ), pass_through_enabled=_env_flag( env, "QDL_STABLE_PASS_THROUGH_ENABLED", default=False ), @@ -383,6 +433,15 @@ def stable_uvicorn_tls(config: StableRuntimeConfig) -> dict[str, object]: } +def stable_request_bounds(config: StableRuntimeConfig) -> RequestBounds: + """Keep query and internal stream ingress on one configured deadline.""" + return RequestBounds( + max_request_bytes=config.max_request_bytes, + request_deadline_seconds=config.request_deadline_seconds, + max_concurrent_requests=config.max_concurrent_requests, + ) + + def stable_grpc_server_credentials( config: StableRuntimeConfig, ) -> grpc.ServerCredentials: @@ -412,11 +471,14 @@ def build_stable_identity( return DataPlaneIdentityService(security, manifests, quota=quota) -def build_stable_spool(config: StableRuntimeConfig) -> SQLiteDurableSpool: +def build_stable_spool( + config: StableRuntimeConfig, catalog: StableSourceCatalog +) -> SQLiteDurableSpool: config.durable_state_dir.mkdir(parents=True, exist_ok=True) + capacity = stable_spool_capacity(catalog) return SQLiteDurableSpool(SpoolConfig( path=config.durable_state_dir / "canonical-cache.sqlite3", - max_records=1_000_000, + max_records=capacity.max_records, max_payload_bytes=2 * 1024 * 1024 * 1024, max_storage_bytes=3 * 1024 * 1024 * 1024, max_partitions=100_000, @@ -424,7 +486,9 @@ def build_stable_spool(config: StableRuntimeConfig) -> SQLiteDurableSpool: min_free_disk_bytes=512 * 1024 * 1024, consumer_ttl_seconds=config.cursor_ttl_seconds, replay_retention_seconds=24 * 3600, - max_partition_records=10_000, + max_partition_records=capacity.max_partition_records, + retain_partition_windows=True, + verify_integrity_on_open=False, )) @@ -451,10 +515,13 @@ def stable_readiness( extra_probes=(), ) -> MeasuredRuntimeReadiness: async def cache(): - stats = await asyncio.to_thread(spool.stats) + summary = await asyncio.to_thread(spool.readiness_summary) return _ready( "query_cache", - detail=f"bounded rebuildable cache records={stats.records} utilization={stats.utilization:.6f}", + detail=( + "bounded rebuildable cache readable " + f"records={summary.records} payload_bytes={summary.payload_bytes}" + ), ) async def redis_probe(): @@ -517,8 +584,8 @@ def create_stable_query_app(config: StableRuntimeConfig | None = None) -> FastAP config.state_dir.mkdir(parents=True, exist_ok=True) manifests = load_stable_manifests(config) identity = build_stable_identity(config, manifests) - spool = build_stable_spool(config) catalog = StableSourceCatalog.load(config.source_bindings_path) + spool = build_stable_spool(config, catalog) handoff = build_stable_handoff(config, spool) service, _backend, issuer = build_stable_query_stack( spool=spool, catalog=catalog, schema_digest=config.schema_digest, @@ -539,11 +606,7 @@ def create_stable_query_app(config: StableRuntimeConfig | None = None) -> FastAP app = create_v2_app( service, identity_service=identity, readiness_service=readiness, cursor_issuer=issuer, - request_bounds=RequestBounds( - max_request_bytes=config.max_request_bytes, - request_deadline_seconds=config.request_deadline_seconds, - max_concurrent_requests=config.max_concurrent_requests, - ), + request_bounds=stable_request_bounds(config), contract_version="2.0.0", authority="INTERNAL_STABLE", ) app.state.runtime_manifest = config.public_manifest() @@ -596,9 +659,9 @@ def create_stable_stream_runtime( config.state_dir.mkdir(parents=True, exist_ok=True) manifests = load_stable_manifests(config) identity = build_stable_identity(config, manifests) - spool = build_stable_spool(config) - handoff = build_stable_handoff(config, spool) catalog = StableSourceCatalog.load(config.source_bindings_path) + spool = build_stable_spool(config, catalog) + handoff = build_stable_handoff(config, spool) async_redis = AsyncRedis.from_url(config.redis_url, decode_responses=True) lease = ActivePassiveGatewayLease( RedisGatewayLeaseStore(async_redis, prefix=config.redis_prefix), @@ -644,10 +707,7 @@ def create_stable_stream_runtime( ) app.add_middleware( BoundedRequestMiddleware, - bounds=RequestBounds( - max_request_bytes=config.max_request_bytes, - max_concurrent_requests=config.max_concurrent_requests, - ), + bounds=stable_request_bounds(config), ) install_stable_health(app, readiness, config.public_manifest()) install_stable_canonical_ingest( @@ -688,8 +748,8 @@ async def serve_stable_projector() -> None: config = StableRuntimeConfig.from_environment("projector_v2") config.state_dir.mkdir(parents=True, exist_ok=True) manifests = load_stable_manifests(config) - spool = build_stable_spool(config) catalog = StableSourceCatalog.load(config.source_bindings_path) + spool = build_stable_spool(config, catalog) assert config.kafka_cert_root is not None assert config.kafka_bootstrap_servers is not None assert config.kafka_client_id is not None @@ -723,11 +783,11 @@ async def serve_stable_projector() -> None: namespace=config.redis_prefix.rstrip(":"), dedicated_database=True, ) - spool_stats = await asyncio.to_thread(spool.stats) + spool_usage = await asyncio.to_thread(spool.readiness_summary) await asyncio.to_thread( target.bind_cache, spool.cache_id, - initialize_if_missing=spool_stats.records == 0, + initialize_if_missing=spool_usage.records == 0, ) active_broker: list[ConfluentProjectorBroker | None] = [None] @@ -744,7 +804,8 @@ def broker_factory(): target=target, max_pending_records=config.max_pending_records, max_pending_bytes=config.max_pending_bytes, - max_batch_records=512, + max_batch_records=config.projector_max_batch_records, + max_batch_bytes=config.projector_max_batch_bytes, batch_wait_seconds=0.01, ) diff --git a/qdl/runtime/stable_bar_edge.py b/qdl/runtime/stable_bar_edge.py index 9d6767f7..673aa371 100644 --- a/qdl/runtime/stable_bar_edge.py +++ b/qdl/runtime/stable_bar_edge.py @@ -9,6 +9,7 @@ import sqlite3 import threading import time +from dataclasses import dataclass from pathlib import Path from qdl.adapters.intervals import ( @@ -29,6 +30,10 @@ from qdl.common.v1 import common_pb2 from qdl.marketdata.v2 import market_data_pb2 from qdl.runtime.stable_catalog import StableSourceBinding, StableSourceCatalog +from qdl.runtime.stable_capacity import ( + STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, + STABLE_SPOOL_PUBLIC_PARTITION_WINDOW, +) from qdl.runtime.stable_deployment import ( StableAcquisitionBinding, StableAcquisitionPlan, @@ -39,6 +44,9 @@ logger = logging.getLogger(__name__) +_MAX_DURABLE_BAR_ROWS = STABLE_SPOOL_PUBLIC_PARTITION_WINDOW +_DURABLE_COVERAGE_BATCH_ROWS = 256 + _STATE_SCHEMA_V2 = "qdl.stable-bar-edge-state.v2" _STATE_SCHEMA_V3 = "qdl.stable-bar-edge-state.v3" @@ -53,6 +61,23 @@ _BOOTSTRAP_HISTORY_LOOKBACK_MS = _BOOTSTRAP_HISTORY_LOOKBACK_DAYS * 86_400_000 +@dataclass(frozen=True, slots=True) +class StableBarHistoryRepairPlan: + """One verified, provider-backed final-BAR recovery window. + + The plan deliberately contains provider envelopes instead of writing cache + state directly. Applying it must still travel through raw Kafka, the Rust + canonical core and projectors, exactly like normal market data. + """ + + source: StableSourceBinding + acquisition: StableAcquisitionBinding + envelopes: tuple[object, ...] + opens: tuple[int, ...] + expected_opens: frozenset[int] + missing_envelopes: tuple[object, ...] + + def _canonical_cache_id(path: str | Path) -> str: """Read the durable cache generation without initializing or mutating it.""" database = Path(path).expanduser().resolve() @@ -82,6 +107,12 @@ def _bar_interval_ms(interval: str) -> int: raise ValueError(f"stable BAR interval is unsupported: {interval}") from error +def durable_bar_history_capacity_rows(interval: str) -> int: + """Return the truthful retained-row ceiling for one durable BAR interval.""" + + return max(1, _BOOTSTRAP_HISTORY_LOOKBACK_MS // _bar_interval_ms(interval)) + + def _source_provider(source: StableSourceBinding) -> str: """Return the catalog venue used for provider BAR calendar alignment.""" instrument = getattr(source, "instrument", None) @@ -136,13 +167,18 @@ def __init__( state_path: str | Path | None = None, canonical_cache_id: str | None = None, canonical_cache_path: str | Path | None = None, + repair_only: bool = False, clock=time.time, generation_clock_ns=time.time_ns, ) -> None: - if not 1 <= warmup_rows <= 1000: - raise ValueError("stable BAR warmup rows must be between 1 and 1000") - if not 1 <= max_catchup_rows <= 1000: - raise ValueError("stable BAR catch-up rows must be between 1 and 1000") + if not 1 <= warmup_rows <= _MAX_DURABLE_BAR_ROWS: + raise ValueError( + "stable BAR warmup rows must be between 1 and 10000" + ) + if not 1 <= max_catchup_rows <= _MAX_DURABLE_BAR_ROWS: + raise ValueError( + "stable BAR catch-up rows must be between 1 and 10000" + ) if not 0.01 <= settlement_delay_seconds <= 2.0: raise ValueError("stable BAR initial poll delay must be between 0.01 and 2 seconds") if not 0.01 <= final_retry_initial_seconds <= final_retry_max_seconds <= 5.0: @@ -160,6 +196,9 @@ def __init__( self.final_retry_max_seconds = final_retry_max_seconds self.max_concurrent_requests = max_concurrent_requests self.state_path = Path(state_path) if state_path is not None else None + self.repair_only = repair_only + if repair_only and (self.state_path is None or not self.state_path.is_file()): + raise RuntimeError("stable BAR repair requires the active writer checkpoint") self.canonical_cache_id = ( str(canonical_cache_id).strip().lower() if canonical_cache_id is not None @@ -252,11 +291,16 @@ def __init__( self._rest_fallback_active = bool(self.bindings or self.okx_bindings) if self._history_bootstrap_active: self._restore_state() - self._issue_connection_generation() - # Persist the new generation before any provider row can be sent to - # Kafka. A crash after issuance must never reuse an older source - # generation on restart. - self._persist_state() + if self.repair_only: + if self.connection_generation <= 0: + raise RuntimeError("stable BAR repair requires a versioned writer generation") + self._set_session_ids() + self._assert_repair_writer_current() + else: + self._issue_connection_generation() + # Persist before publishing, so a restart cannot reuse an + # older writer generation. Repairs never claim a generation. + self._persist_state() else: self._history_bootstrapped = True @@ -305,6 +349,9 @@ def _issue_connection_generation(self) -> None: self.connection_generation = max(previous + 1, floor) if self.connection_generation > _MAX_CONNECTION_GENERATION: raise RuntimeError("stable BAR connection generation is exhausted") + self._set_session_ids() + + def _set_session_ids(self) -> None: suffix = f"r{self._authority_revision}-g{self.connection_generation}" self.binance_session_id = f"qdl-v2-stable-binance-rest-{suffix}" self.okx_session_id = f"qdl-v2-stable-okx-rest-{suffix}" @@ -390,6 +437,16 @@ def _restore_state(self) -> None: "stable BAR checkpoint cache generation changed; bounded bootstrap required" ) return + gaps = {} if self.repair_only else self._checkpoint_history_gaps(restored) + if gaps: + for binding_id in gaps: + restored.pop(binding_id, None) + logger.warning( + "stable BAR checkpoint history incomplete; bounded repair required " + "bindings=%s missing_rows=%s", + ",".join(sorted(gaps)), + sum(gaps.values()), + ) self._last_open_ms = restored self._history_bootstrapped = set(restored) == set(self._binding_ids) logger.info( @@ -400,6 +457,8 @@ def _restore_state(self) -> None: ) def _persist_state(self) -> None: + if self.repair_only: + raise RuntimeError("stable BAR repair cannot write the active checkpoint") if self.state_path is None: return parent = self.state_path.parent @@ -441,6 +500,40 @@ def _persist_state(self) -> None: os.close(descriptor) temporary.unlink(missing_ok=True) + def _checkpoint_history_gaps( + self, + restored: dict[str, int], + ) -> dict[str, int]: + """Return incomplete warmup windows behind a matching checkpoint. + + A checkpoint watermark proves only that the edge previously observed a + final close. It cannot prove that the durable cache still retains the + bounded history needed by a V2 warmup after cache rebuild or compaction. + This validation is cache-only; deficient bindings re-bootstrap through + the normal real-provider pipeline on the next edge loop. + """ + + if self.canonical_cache_path is None: + return {} + sources = { + source.binding_id: source + for source, _acquisition in self.history_bindings + self.history_okx_bindings + } + gaps: dict[str, int] = {} + for binding_id, last_open_ms in restored.items(): + source = sources[binding_id] + interval_ms = _bar_interval_ms(source.interval or "") + expected_opens = frozenset( + last_open_ms - index * interval_ms + for index in range(self._bootstrap_rows_for(source)) + ) + missing = len(expected_opens - self._durable_final_bar_opens( + source, expected_opens + )) + if missing: + gaps[binding_id] = missing + return gaps + def _binance_binding( self, source: StableSourceBinding, @@ -483,14 +576,14 @@ def _okx_binding( instrument_catalog_revision=self.catalog.catalog_revision, ) - def _publish_history( + def _history_repair_plan( self, source: StableSourceBinding, acquisition: StableAcquisitionBinding, envelopes, *, expected_rows: int, - ) -> int: + ) -> StableBarHistoryRepairPlan: values = tuple(envelopes) if len(values) != expected_rows: raise RuntimeError( @@ -508,32 +601,212 @@ def _publish_history( item for item, open_ms in zip(values, opens, strict=True) if open_ms not in existing_opens ) - acknowledgements = self.publisher.publish_many(missing) if missing else () - if len(acknowledgements) != len(missing): + return StableBarHistoryRepairPlan( + source=source, + acquisition=acquisition, + envelopes=values, + opens=opens, + expected_opens=expected_opens, + missing_envelopes=missing, + ) + + def _apply_history_repair_plan( + self, + plan: StableBarHistoryRepairPlan, + *, + advance_watermark: bool, + expected_missing_rows: int | None = None, + revalidate_missing_rows: bool = False, + ) -> int: + self._assert_repair_writer_current() + if self.repair_only and advance_watermark: + raise RuntimeError("stable BAR repair cannot advance the writer watermark") + current = plan + if revalidate_missing_rows: + current = self._history_repair_plan( + plan.source, + plan.acquisition, + plan.envelopes, + expected_rows=len(plan.envelopes), + ) + if current.expected_opens != plan.expected_opens: + raise RuntimeError("stable BAR recovery window changed before publish") + if ( + expected_missing_rows is not None + and len(current.missing_envelopes) != expected_missing_rows + ): + raise RuntimeError( + "stable BAR recovery missing-row count changed before publish " + f"binding={plan.source.binding_id} expected={expected_missing_rows} " + f"actual={len(current.missing_envelopes)}" + ) + self._assert_repair_writer_current() + acknowledgements = ( + self.publisher.publish_many(current.missing_envelopes) + if current.missing_envelopes + else () + ) + if len(acknowledgements) != len(current.missing_envelopes): raise RuntimeError("stable BAR bootstrap did not receive every Kafka ACK") published_opens = { - self._open_time_ms(acquisition, item) for item in missing + self._open_time_ms(plan.acquisition, item) + for item in current.missing_envelopes } - if existing_opens | published_opens != expected_opens: + if self._durable_final_bar_opens( + plan.source, plan.expected_opens + ) | published_opens != plan.expected_opens: raise RuntimeError( - f"stable BAR bootstrap did not cover every durable open binding={source.binding_id}" + f"stable BAR bootstrap did not cover every durable open binding={plan.source.binding_id}" ) self._assert_canonical_cache_identity() - self._last_open_ms[source.binding_id] = max(expected_opens) - self._persist_state() + if advance_watermark: + self._last_open_ms[plan.source.binding_id] = max(plan.expected_opens) + self._persist_state() logger.info( - "stable real-provider BAR bootstrap ACK binding=%s venue=%s expected_rows=%s " - "published_rows=%s existing_durable_rows=%s first_open_ms=%s last_open_ms=%s", - source.binding_id, - acquisition.runtime, - len(values), - len(missing), - len(existing_opens), - min(expected_opens), - max(expected_opens), + "stable provider final BAR history ACK binding=%s venue=%s expected_rows=%s " + "published_rows=%s existing_durable_rows=%s advance_watermark=%s", + plan.source.binding_id, + plan.acquisition.runtime, + len(plan.envelopes), + len(acknowledgements), + len(plan.expected_opens) - len(current.missing_envelopes), + advance_watermark, ) return len(acknowledgements) + def _assert_repair_writer_current(self) -> None: + if not self.repair_only: + return + self._assert_canonical_cache_identity() + try: + checkpoint = json.loads(self.state_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise RuntimeError("stable BAR repair writer checkpoint is unavailable") from error + expected = self._state_payload() + if not isinstance(checkpoint, dict) or any( + checkpoint.get(key) != value + for key, value in expected.items() if key != "last_open_ms" + ): + raise RuntimeError("stable BAR repair writer generation or identity changed") + + def _publish_history( + self, + source: StableSourceBinding, + acquisition: StableAcquisitionBinding, + envelopes, + *, + expected_rows: int, + ) -> int: + return self._apply_history_repair_plan( + self._history_repair_plan( + source, + acquisition, + envelopes, + expected_rows=expected_rows, + ), + advance_watermark=True, + ) + + def _fetch_history( + self, + source: StableSourceBinding, + acquisition: StableAcquisitionBinding, + *, + rows: int, + observed_ms: int, + ) -> tuple[object, ...]: + if acquisition.runtime == "BINANCE": + return tuple(fetch_binance_history( + self._binance_binding(source), + limit=rows, + now_ms=observed_ms, + attempts=4, + test_provenance=False, + )) + if acquisition.runtime == "OKX": + return tuple(asyncio.run(fetch_okx_history( + self._okx_binding(source), + limit=rows, + now_ms=observed_ms, + test_provenance=False, + ))) + raise ValueError("stable crypto BAR runtime is unsupported") + + def prepare_history_repair( + self, + binding_id: str, + *, + rows: int, + observed_ms: int | None = None, + ) -> StableBarHistoryRepairPlan: + """Return one provider-confirmed repair plan without writing state.""" + + if not 1 <= rows <= _MAX_DURABLE_BAR_ROWS: + raise ValueError("stable BAR repair rows must be between 1 and 10000") + pairs = { + source.binding_id: (source, acquisition) + for source, acquisition in self.history_bindings + self.history_okx_bindings + } + try: + source, acquisition = pairs[binding_id] + except KeyError as error: + raise ValueError( + f"stable BAR repair binding is not enabled history demand: {binding_id}" + ) from error + # A repair is a bounded recovery operation, not an ordinary service + # warmup. The active edge may intentionally retain a smaller startup + # warmup (for example 500 rows), while a verified historical hole can + # require a larger provider-backed window. Keep the repair bounded by + # the same truthful interval capacity and global public maximum, but + # never couple it to the live loop's configured warmup size. + maximum_rows = min( + _MAX_DURABLE_BAR_ROWS, + durable_bar_history_capacity_rows(source.interval or ""), + ) + if rows > maximum_rows: + raise ValueError( + f"stable BAR repair rows exceed durable provider window binding={binding_id} " + f"requested={rows} maximum={maximum_rows}" + ) + return self._history_repair_plan( + source, + acquisition, + self._fetch_history( + source, + acquisition, + rows=rows, + observed_ms=( + self._settled_observed_ms() + if observed_ms is None + else observed_ms + ), + ), + expected_rows=rows, + ) + + def apply_history_repair( + self, + plan: StableBarHistoryRepairPlan, + *, + expected_missing_rows: int, + ) -> int: + """Publish only a prepared window's still-missing provider final bars.""" + + if expected_missing_rows < 0: + raise ValueError("stable BAR repair expected missing rows must be non-negative") + return self._apply_history_repair_plan( + plan, + advance_watermark=False, + expected_missing_rows=expected_missing_rows, + revalidate_missing_rows=True, + ) + + def history_repair_remaining_rows(self, plan: StableBarHistoryRepairPlan) -> int: + """Return the authoritative missing-open count for a prepared window.""" + + covered = self._durable_final_bar_opens(plan.source, plan.expected_opens) + return len(plan.expected_opens - covered) + def _assert_canonical_cache_identity(self) -> None: """Refuse to certify a bootstrap if its durable generation changed.""" if self.canonical_cache_path is None: @@ -542,6 +815,40 @@ def _assert_canonical_cache_identity(self) -> None: if _canonical_cache_id(self.canonical_cache_path) != self.canonical_cache_id: raise RuntimeError("stable BAR canonical cache generation changed during bootstrap") + def _rebase_if_canonical_cache_generation_changed(self) -> bool: + """Re-bootstrap from providers after an external canonical-cache rebuild. + + The edge checkpoint and canonical cache are separate durable artifacts. + If a cache rebuild happens while this process remains alive, a watermark + alone cannot prove that the rebuilt cache still contains its warmup + history. Rebase exactly as a fresh edge process would: clear only this + edge's watermarks, issue a new provider-session generation, and let the + existing bounded provider-history bootstrap replenish the cache. + """ + if self.canonical_cache_path is None: + return False + assert self.canonical_cache_id is not None + observed_cache_id = _canonical_cache_id(self.canonical_cache_path) + if observed_cache_id == self.canonical_cache_id: + return False + + previous_cache_id = self.canonical_cache_id + self.canonical_cache_id = observed_cache_id + self._last_open_ms.clear() + self._retry_attempts.clear() + self._next_retry_at.clear() + self._last_retry_log.clear() + self._history_bootstrapped = False + self._issue_connection_generation() + self._persist_state() + logger.warning( + "stable BAR canonical cache generation changed; rebased checkpoint " + "previous=%s current=%s", + previous_cache_id, + observed_cache_id, + ) + return True + def _durable_final_bar_opens( self, source: StableSourceBinding, @@ -559,23 +866,27 @@ def _durable_final_bar_opens( if not expected_opens or self.canonical_cache_path is None: return frozenset() self._assert_canonical_cache_identity() + connection = None try: connection = sqlite3.connect( f"{self.canonical_cache_path.as_uri()}?mode=ro", uri=True ) - try: - rows = connection.execute( - """ - SELECT payload FROM events - WHERE stream = ? AND partition_key = ? - ORDER BY logical_offset DESC - LIMIT 10000 - """, - (source.canonical_stream, source.partition_key), - ).fetchall() - finally: - connection.close() + cursor = connection.execute( + """ + SELECT payload FROM events + WHERE stream = ? AND partition_key = ? + ORDER BY logical_offset DESC + LIMIT ? + """, + ( + source.canonical_stream, + source.partition_key, + STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, + ), + ) except (OSError, sqlite3.Error) as error: + if connection is not None: + connection.close() raise RuntimeError("stable BAR durable coverage is unavailable") from error covered: set[int] = set() @@ -584,34 +895,40 @@ def _durable_final_bar_opens( market_data_pb2.BAR_LIFECYCLE_REVISED, } expected_role = getattr(common_pb2, f"SOURCE_ROLE_{source.source_role}") - for (payload,) in rows: - try: - envelope = market_data_pb2.EventEnvelope.FromString(payload) - except Exception as error: - raise RuntimeError("stable BAR durable payload is unreadable") from error - if envelope.WhichOneof("payload") != "bar": - raise RuntimeError("stable BAR durable partition contains a non-BAR payload") - if ( - envelope.instrument_uid != source.instrument.instrument_uid - or envelope.instrument_id != source.instrument.instrument_id - or not source.accepts_instrument_revision(envelope.instrument_revision) - or envelope.venue != source.instrument.identity.venue - or envelope.market != source.instrument.identity.market - or envelope.product_type != source.instrument.identity.product_type.value - or envelope.native_symbol != source.instrument.native_symbol - or envelope.provider != source.provider - or envelope.source_id != source.source_id - or envelope.source_role != expected_role - or envelope.bar.interval != source.interval - ): - raise RuntimeError("stable BAR durable partition differs from its binding") - open_ms = int(envelope.bar.open_time_ns) // 1_000_000 - if ( - open_ms in expected_opens - and envelope.bar.is_final - and envelope.bar.lifecycle in final_lifecycles - ): - covered.add(open_ms) + try: + while rows := cursor.fetchmany(_DURABLE_COVERAGE_BATCH_ROWS): + for (payload,) in rows: + try: + envelope = market_data_pb2.EventEnvelope.FromString(payload) + except Exception as error: + raise RuntimeError("stable BAR durable payload is unreadable") from error + if envelope.WhichOneof("payload") != "bar": + raise RuntimeError("stable BAR durable partition contains a non-BAR payload") + if ( + envelope.instrument_uid != source.instrument.instrument_uid + or envelope.instrument_id != source.instrument.instrument_id + or not source.accepts_instrument_revision(envelope.instrument_revision) + or envelope.venue != source.instrument.identity.venue + or envelope.market != source.instrument.identity.market + or envelope.product_type != source.instrument.identity.product_type.value + or envelope.native_symbol != source.instrument.native_symbol + or envelope.provider != source.provider + or envelope.source_id != source.source_id + or envelope.source_role != expected_role + or envelope.bar.interval != source.interval + ): + raise RuntimeError("stable BAR durable partition differs from its binding") + open_ms = int(envelope.bar.open_time_ns) // 1_000_000 + if ( + open_ms in expected_opens + and envelope.bar.is_final + and envelope.bar.lifecycle in final_lifecycles + ): + covered.add(open_ms) + if covered == expected_opens: + return frozenset(covered) + finally: + connection.close() self._assert_canonical_cache_identity() return frozenset(covered) @@ -619,15 +936,14 @@ def _bootstrap_rows_for(self, source: StableSourceBinding) -> int: """Return the real-history bound for one fixed-duration BAR. `warmup_rows` stays a global upper bound, but a weekly provider request - for 1,000 rows would require roughly nineteen years that neither + for 10,000 rows would require roughly 192 years that neither Binance nor OKX can truthfully supply. The interval-aware cap makes a long BAR bootstrap bounded and honest while keeping minute/hour warmups at the configured maximum. """ - interval_ms = _bar_interval_ms(source.interval or "") return min( self.warmup_rows, - max(1, _BOOTSTRAP_HISTORY_LOOKBACK_MS // interval_ms), + durable_bar_history_capacity_rows(source.interval or ""), ) def _settled_observed_ms(self) -> int: @@ -707,6 +1023,9 @@ def _retry_is_due(self, binding_id: str, *, now: float) -> bool: return getattr(self, "_next_retry_at", {}).get(binding_id, 0.0) <= now def bootstrap_history(self) -> int: + if self.repair_only: + raise RuntimeError("stable BAR repair cannot bootstrap as writer") + self._rebase_if_canonical_cache_generation_changed() if not self._history_bootstrap_active: return 0 if self._history_bootstrapped: @@ -720,12 +1039,11 @@ def bootstrap_history(self) -> int: published += self._publish_history( source, acquisition, - fetch_binance_history( - self._binance_binding(source), - limit=bootstrap_rows, - now_ms=observed_ms, - attempts=4, - test_provenance=False, + self._fetch_history( + source, + acquisition, + rows=bootstrap_rows, + observed_ms=observed_ms, ), expected_rows=bootstrap_rows, ) @@ -736,12 +1054,12 @@ def bootstrap_history(self) -> int: published += self._publish_history( source, acquisition, - asyncio.run(fetch_okx_history( - self._okx_binding(source), - limit=bootstrap_rows, - now_ms=observed_ms, - test_provenance=False, - )), + self._fetch_history( + source, + acquisition, + rows=bootstrap_rows, + observed_ms=observed_ms, + ), expected_rows=bootstrap_rows, ) self._history_bootstrapped = ( @@ -900,6 +1218,8 @@ def _fetch_latest( raise ValueError("stable crypto BAR runtime is unsupported") def run_cycle(self) -> int: + if self.repair_only: + raise RuntimeError("stable BAR repair cannot run the writer cycle") if not self._rest_fallback_active: return 0 observed_ms = self._settled_observed_ms() @@ -1005,6 +1325,8 @@ def _loop_sleep_seconds(self, now: float) -> float: return max(0.01, self._next_ready_at(now) - now) def run_forever(self) -> None: + if self.repair_only: + raise RuntimeError("stable BAR repair cannot run the writer loop") if not self._history_bootstrap_active: logger.info("stable crypto BAR edge idle; no enabled crypto BAR demand") while not self._stopped.wait(60.0): @@ -1037,7 +1359,14 @@ def stop(self, *_args) -> None: self.publisher.close() -def build_from_environment() -> StableBinanceBarEdge: +def build_from_environment( + *, + state_path: str | Path | None = None, + client_id: str | None = None, + repair_only: bool = False, +) -> StableBinanceBarEdge: + """Build the shared edge or an isolated repair client from one runtime env.""" + catalog = StableSourceCatalog.load(os.environ["QDL_STABLE_SOURCE_BINDINGS"]) acquisition = StableAcquisitionPlan.load( os.environ["QDL_STABLE_ACQUISITION_BINDINGS"], catalog=catalog @@ -1047,7 +1376,7 @@ def build_from_environment() -> StableBinanceBarEdge: cert_root = Path(os.environ["QDL_KAFKA_CERT_ROOT"]) publisher = KafkaRawPublisher(KafkaRawPublisherConfig( bootstrap_servers=os.environ["QDL_KAFKA_BOOTSTRAP_SERVERS"], - client_id=os.environ["QDL_KAFKA_CLIENT_ID"], + client_id=client_id or os.environ["QDL_KAFKA_CLIENT_ID"], topic=acquisition.raw_topic, ca_path=cert_root / "ca.crt", certificate_path=cert_root / "client.crt", @@ -1086,7 +1415,7 @@ def build_from_environment() -> StableBinanceBarEdge: max_concurrent_requests=int( os.environ.get("QDL_STABLE_BAR_MAX_CONCURRENT_REQUESTS", "32") ), - state_path=os.environ.get( + state_path=state_path or os.environ.get( "QDL_STABLE_BAR_STATE_PATH", str( Path( @@ -1100,6 +1429,7 @@ def build_from_environment() -> StableBinanceBarEdge: ), canonical_cache_id=_canonical_cache_id(canonical_cache_path), canonical_cache_path=canonical_cache_path, + repair_only=repair_only, ) diff --git a/qdl/runtime/stable_capacity.py b/qdl/runtime/stable_capacity.py new file mode 100644 index 00000000..3f1f0af1 --- /dev/null +++ b/qdl/runtime/stable_capacity.py @@ -0,0 +1,13 @@ +"""Shared bounded capacities for stable V2 durable BAR retention.""" + +from __future__ import annotations + + +# Public callers may request at most this many historical BARs. The durable +# spool retains a small additional tail so late authentic backfills do not +# evict an otherwise required public history window by append order. +STABLE_SPOOL_PUBLIC_PARTITION_WINDOW = 10_000 +STABLE_SPOOL_LATE_BACKFILL_HEADROOM = 64 +STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW = ( + STABLE_SPOOL_PUBLIC_PARTITION_WINDOW + STABLE_SPOOL_LATE_BACKFILL_HEADROOM +) diff --git a/qdl/runtime/stable_deployment.py b/qdl/runtime/stable_deployment.py index 5dac5b40..9ed03cc2 100644 --- a/qdl/runtime/stable_deployment.py +++ b/qdl/runtime/stable_deployment.py @@ -17,6 +17,7 @@ _SEQUENCE_POLICIES = frozenset({"NONE", "MONOTONIC", "CONTIGUOUS"}) STABLE_TOPIC_PARTITIONS = 6 STABLE_CORE_WORKER_COUNT = 3 +STABLE_CORE_DEDUP_CAPACITY = 100_000 V2_REALTIME_RAW_TOPIC = "md.raw.realtime.v2" SHARED_REALTIME_CORE_GROUP_ID = "qdl-v2-realtime-core-v2" SHARED_REALTIME_CORE_ID_PREFIX = "qdl-v2-realtime-core" @@ -58,6 +59,7 @@ class StableL2Acquisition: depth_per_side: int rest_snapshot_url: str | None snapshot_refresh_seconds: int | None + materialized_snapshot_interval_ms: int | None = None def validate(self, source: StableSourceBinding, acquisition: "StableAcquisitionBinding") -> None: if source.feed not in {FeedType.BOOK_SNAPSHOT, FeedType.BOOK_DELTA}: @@ -66,6 +68,10 @@ def validate(self, source: StableSourceBinding, acquisition: "StableAcquisitionB raise ValueError("L2 depth is outside bounded range") if acquisition.mode != "RUST_NATIVE" or acquisition.sequence_policy != "CONTIGUOUS": raise ValueError("L2 acquisition requires lossless Rust-native continuity") + if self.materialized_snapshot_interval_ms is not None and not ( + 100 <= self.materialized_snapshot_interval_ms <= 60_000 + ): + raise ValueError("L2 materialized snapshot cadence is outside bounded range") if self.provider_protocol == "BINANCE_DIFF_DEPTH": if ( source.instrument.identity.venue != "BINANCE" @@ -100,11 +106,16 @@ def validate(self, source: StableSourceBinding, acquisition: "StableAcquisitionB raise ValueError("L2 provider protocol is not certified") def core_mapping(self) -> dict[str, object]: - return { + result: dict[str, object] = { "provider_protocol": self.provider_protocol, "depth_per_side": self.depth_per_side, "snapshot_refresh_seconds": self.snapshot_refresh_seconds, } + if self.materialized_snapshot_interval_ms is not None: + result["materialized_snapshot_interval_ms"] = ( + self.materialized_snapshot_interval_ms + ) + return result def validate_shared_authority_record(authority: Mapping[str, Any]) -> None: @@ -343,10 +354,16 @@ def load( raise ValueError("stable acquisition 'enabled' must be a boolean") l2_raw = value.get("l2") if l2_raw is not None: - if not isinstance(l2_raw, dict) or set(l2_raw) != { + required_l2 = { "provider_protocol", "depth_per_side", "rest_snapshot_url", "snapshot_refresh_seconds", - }: + } + optional_l2 = {"materialized_snapshot_interval_ms"} + if ( + not isinstance(l2_raw, dict) + or not required_l2 <= set(l2_raw) + or set(l2_raw) - required_l2 - optional_l2 + ): raise ValueError("stable L2 acquisition fields are incomplete or unknown") l2 = StableL2Acquisition( provider_protocol=str(l2_raw["provider_protocol"]).upper(), @@ -361,6 +378,11 @@ def load( if l2_raw["snapshot_refresh_seconds"] is not None else None ), + materialized_snapshot_interval_ms=( + int(l2_raw["materialized_snapshot_interval_ms"]) + if l2_raw.get("materialized_snapshot_interval_ms") is not None + else None + ), ) else: l2 = None @@ -538,7 +560,9 @@ def core_config( "canonical_stream": self.canonical_topic, "quarantine_stream": self.quarantine_topic, "allow_test_provenance": False, - "dedup_capacity": 1_000_000, + # Kafka transactions atomically commit output and raw offsets; + # this is a bounded in-process replay guard, not durable state. + "dedup_capacity": STABLE_CORE_DEDUP_CAPACITY, "bindings": bindings, }, "raw_topics": [self.raw_topic], diff --git a/qdl/runtime/stable_ingest.py b/qdl/runtime/stable_ingest.py index 93607528..53a82523 100644 --- a/qdl/runtime/stable_ingest.py +++ b/qdl/runtime/stable_ingest.py @@ -9,6 +9,7 @@ import re import ssl import uuid +from collections.abc import Iterator from dataclasses import dataclass, field from urllib.parse import urlsplit @@ -22,7 +23,7 @@ from qdl.runtime.lease import GatewayFenced from qdl.runtime.stable_catalog import StableSourceCatalog from qdl.stream import DurableStreamGateway -from qdl.transport import DurableEvent, SQLiteDurableSpool, StoredEvent +from qdl.transport import BackpressureRequired, DurableEvent, SQLiteDurableSpool, StoredEvent _INGEST_SCHEMA = "qdl.v2.stable-canonical-ingest.v1" @@ -194,6 +195,11 @@ async def ingest( ) except GatewayFenced as error: raise HTTPException(status_code=409, detail="stable gateway was fenced") from error + except BackpressureRequired as error: + raise HTTPException( + status_code=503, + detail="stable canonical cache capacity temporarily unavailable", + ) from error duplicate_ids = [ event.event_id for (_binding, _envelope, event), stored in zip( @@ -269,9 +275,8 @@ async def publish_many( values = tuple(events) if not 1 <= len(values) <= 1000: raise ValueError("stable HTTP sink batch must contain 1..1000 events") - encoded = tuple(self._encode_event(event) for event in values) stored_values = [] - for chunk_values, chunk_encoded in self._request_chunks(values, encoded): + for chunk_values, chunk_encoded in self._request_chunks(values): stored_values.extend( await self._publish_chunk(chunk_values, chunk_encoded) ) @@ -301,34 +306,53 @@ def _body(encoded: tuple[dict[str, str], ...]) -> bytes: "events": encoded, }, sort_keys=True, separators=(",", ":")).encode() + @staticmethod + def _empty_body_size() -> int: + """Return the exact fixed JSON envelope size without retaining a batch.""" + + return len(json.dumps({ + "schema": _INGEST_SCHEMA, + # UUID textual length is fixed; this avoids making a trial body for + # every candidate event while preserving the actual wire bound. + "batch_id": "00000000-0000-4000-8000-000000000000", + "events": (), + }, sort_keys=True, separators=(",", ":")).encode()) + + @staticmethod + def _encoded_item_size(item: dict[str, str]) -> int: + return len(json.dumps(item, sort_keys=True, separators=(",", ":")).encode()) + def _request_chunks( - self, - values: tuple[DurableEvent, ...], - encoded: tuple[dict[str, str], ...], - ) -> tuple[tuple[tuple[DurableEvent, ...], tuple[dict[str, str], ...]], ...]: - chunks = [] + self, values: tuple[DurableEvent, ...], + ) -> Iterator[tuple[tuple[DurableEvent, ...], tuple[dict[str, str], ...]]]: + """Yield exact wire-bounded chunks without O(n^2) trial JSON bodies.""" + current_values: list[DurableEvent] = [] current_encoded: list[dict[str, str]] = [] - for event, item in zip(values, encoded, strict=True): - candidate = (*current_encoded, item) - if len(self._body(candidate)) > self.max_request_bytes: + current_size = self._empty_body_size() + for event in values: + item = self._encode_event(event) + item_size = self._encoded_item_size(item) + candidate_size = current_size + item_size + (1 if current_values else 0) + if candidate_size > self.max_request_bytes: if not current_values: raise ValueError( "stable canonical event exceeds request byte bound" ) - chunks.append((tuple(current_values), tuple(current_encoded))) + yield tuple(current_values), tuple(current_encoded) current_values = [event] current_encoded = [item] - if len(self._body(tuple(current_encoded))) > self.max_request_bytes: + current_size = self._empty_body_size() + item_size + if current_size > self.max_request_bytes: raise ValueError( "stable canonical event exceeds request byte bound" ) continue current_values.append(event) current_encoded.append(item) + current_size = candidate_size if current_values: - chunks.append((tuple(current_values), tuple(current_encoded))) - return tuple(chunks) + yield tuple(current_values), tuple(current_encoded) async def _publish_chunk( self, diff --git a/qdl/runtime/stable_projector.py b/qdl/runtime/stable_projector.py index 90102898..b0d8cb65 100644 --- a/qdl/runtime/stable_projector.py +++ b/qdl/runtime/stable_projector.py @@ -111,6 +111,7 @@ def __init__( max_pending_records: int = 10_000, max_pending_bytes: int = 256 * 1024 * 1024, max_batch_records: int = 128, + max_batch_bytes: int | None = None, batch_wait_seconds: float = 0.025, ) -> None: if ( @@ -123,6 +124,10 @@ def __init__( raise ValueError("stable projector pending bounds must be positive") if not 1 <= max_batch_records <= 1000 or not 0 < batch_wait_seconds <= 1: raise ValueError("stable projector batch policy is invalid") + if max_batch_bytes is None: + max_batch_bytes = min(8 * 1024 * 1024, max_pending_bytes) + if not 1 <= max_batch_bytes <= max_pending_bytes: + raise ValueError("stable projector batch byte bound is invalid") self.broker = broker self.spool = spool self.catalog = catalog @@ -134,6 +139,7 @@ def __init__( self.max_pending_records = max_pending_records self.max_pending_bytes = max_pending_bytes self.max_batch_records = max_batch_records + self.max_batch_bytes = max_batch_bytes self.batch_wait_seconds = batch_wait_seconds poll_headroom = min(max_batch_records, max(1, max_pending_records // 4)) self._canonical_pause_high_records = max( @@ -151,6 +157,7 @@ def __init__( self._raw_committed = 0 self._canonical_committed = 0 self._duplicate_projections = 0 + self._deferred_record: KafkaProjectorRecord | None = None async def accept(self, record: KafkaProjectorRecord) -> None: await self.accept_many((record,)) @@ -212,7 +219,10 @@ async def _accept_assignment_batch( await self._drain_ready() async def run_once(self, timeout_seconds: float = 1.0) -> bool: - record = await asyncio.to_thread(self.broker.poll, timeout_seconds) + record = self._deferred_record + self._deferred_record = None + if record is None: + record = await asyncio.to_thread(self.broker.poll, timeout_seconds) if record is None: # Another projector replica may have persisted the correlated raw # envelope into the shared cache. Retry bounded local partitions so @@ -220,6 +230,7 @@ async def run_once(self, timeout_seconds: float = 1.0) -> bool: await self._drain_ready() return False records = [record] + batch_bytes = len(record.payload) deadline = asyncio.get_running_loop().time() + self.batch_wait_seconds while len(records) < self.max_batch_records: remaining = deadline - asyncio.get_running_loop().time() @@ -228,7 +239,13 @@ async def run_once(self, timeout_seconds: float = 1.0) -> bool: item = await asyncio.to_thread(self.broker.poll, remaining) if item is None: break + if batch_bytes + len(item.payload) > self.max_batch_bytes: + # Kafka has already delivered this record. Keep it in-order for + # the next bounded drain instead of overfilling a Python batch. + self._deferred_record = item + break records.append(item) + batch_bytes += len(item.payload) await self.accept_many(records) return True diff --git a/qdl/runtime/stable_source.py b/qdl/runtime/stable_source.py index b94d2ccc..8a18f2f4 100644 --- a/qdl/runtime/stable_source.py +++ b/qdl/runtime/stable_source.py @@ -39,6 +39,10 @@ StableSourceCatalog, canonical_payload_interval, ) +from qdl.runtime.stable_capacity import ( + STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, + STABLE_SPOOL_PUBLIC_PARTITION_WINDOW, +) from qdl.runtime.session_liveness import StableSessionLivenessReader from qdl.stream import GrpcSnapshot from qdl.transport import Cursor, SQLiteDurableSpool, StoredEvent @@ -182,10 +186,19 @@ def __init__( ) self._clock_ns = clock_ns + def warmup_is_local(self, requirement: DataRequirement) -> bool: + self.catalog.binding_for(requirement) + return True + def latest(self, requirement: DataRequirement) -> MarketDataItem | None: binding = self.catalog.binding_for(requirement) records = self._records( - requirement, limit=10_000 if requirement.feed is FeedType.BAR else 1 + requirement, + limit=( + STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW + if requirement.feed is FeedType.BAR + else 1 + ), ) if not records: return None @@ -210,7 +223,7 @@ def history(self, requirement: DataRequirement) -> HistoryResult | None: # the bounded retained BAR window before selecting the market-time tail; # selecting logical append offsets first can manufacture a false gap. read_limit = ( - 10_000 + STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW if start_ns is not None or binding.feed is FeedType.BAR else requested ) @@ -282,7 +295,11 @@ def open_gaps(self) -> tuple[GapRecord, ...]: records = tuple(self.spool.read_tail( stream=binding.canonical_stream, partition_key=binding.partition_key, - limit=10_000, + limit=( + STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW + if binding.feed is FeedType.BAR + else STABLE_SPOOL_PUBLIC_PARTITION_WINDOW + ), )) gaps.extend(self._gaps(binding, records)) return tuple(sorted(gaps, key=lambda item: (item.detected_at_ns, item.gap_id))) @@ -293,7 +310,7 @@ def stored_events(self, requirement: DataRequirement) -> tuple[StoredEvent, ...] rows = self._records( requirement, limit=( - 10_000 + STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW if start_ns is not None or binding.feed is FeedType.BAR else requested ), @@ -364,12 +381,12 @@ def _requested_window( start_ns=start_ns, end_ns=end_ns, interval_ns=interval_ns, - max_rows=10_000, + max_rows=STABLE_SPOOL_PUBLIC_PARTITION_WINDOW, ) rows = len(expected_opens) if rows < 1: raise ValueError("warmup time range contains no governed session bars") - if rows > 10_000: + if rows > STABLE_SPOOL_PUBLIC_PARTITION_WINDOW: raise ValueError("stable spool time range exceeds bounded query rows") return rows, start_ns, end_ns, expected_opens @@ -386,7 +403,10 @@ def _records( # explicit; it is not an unbounded recovery scan. physical_limit = limit if binding.feed in {FeedType.BOOK_SNAPSHOT, FeedType.BOOK_DELTA}: - physical_limit = min(10_000, max(limit, limit * 512)) + physical_limit = min( + STABLE_SPOOL_PUBLIC_PARTITION_WINDOW, + max(limit, limit * 512), + ) rows = self.spool.read_tail( stream=binding.canonical_stream, partition_key=binding.partition_key, diff --git a/qdl/security/data_plane.py b/qdl/security/data_plane.py index 8e885cd9..69440117 100644 --- a/qdl/security/data_plane.py +++ b/qdl/security/data_plane.py @@ -6,7 +6,7 @@ import threading import time from dataclasses import dataclass -from enum import StrEnum +from qdl._compat import StrEnum from typing import Mapping, Protocol from redis import Redis diff --git a/qdl/transport/__init__.py b/qdl/transport/__init__.py index 716df3e0..5d77b868 100644 --- a/qdl/transport/__init__.py +++ b/qdl/transport/__init__.py @@ -18,7 +18,12 @@ StreamName, ) from qdl.transport.publisher import DurablePublisher, PublisherState -from qdl.transport.sqlite_spool import SQLiteDurableSpool, SpoolConfig, SpoolStats +from qdl.transport.sqlite_spool import ( + SQLiteDurableSpool, + SpoolConfig, + SpoolReadiness, + SpoolStats, +) __all__ = [ "AppendResult", @@ -38,6 +43,7 @@ "RetryDecision", "SQLiteDurableSpool", "SpoolConfig", + "SpoolReadiness", "SpoolStats", "StoredEvent", "StreamName", diff --git a/qdl/transport/kafka_projector.py b/qdl/transport/kafka_projector.py index ebeec07b..65c431b9 100644 --- a/qdl/transport/kafka_projector.py +++ b/qdl/transport/kafka_projector.py @@ -69,6 +69,9 @@ class KafkaProjectorConfig: max_poll_interval_ms: int = 300_000 checkpoint_batch_size: int = 128 checkpoint_interval_ms: int = 100 + queued_max_messages_kbytes: int = 16 * 1024 + fetch_max_bytes: int = 8 * 1024 * 1024 + max_partition_fetch_bytes: int = 2 * 1024 * 1024 def validate(self) -> None: if not all(( @@ -85,6 +88,10 @@ def validate(self) -> None: 10 <= self.checkpoint_interval_ms <= 5_000 ): raise ValueError("Kafka stable projector checkpoint policy is invalid") + if not 1_024 <= self.queued_max_messages_kbytes <= 64 * 1024: + raise ValueError("Kafka stable projector queue bound is invalid") + if not 1_024 * 1_024 <= self.max_partition_fetch_bytes <= self.fetch_max_bytes <= 32 * 1024 * 1024: + raise ValueError("Kafka stable projector fetch bounds are invalid") for value in (self.ca_path, self.certificate_path, self.key_path): if not value.is_file(): raise ValueError(f"Kafka stable projector TLS file is unavailable: {value}") @@ -121,6 +128,9 @@ def __init__(self, config: KafkaProjectorConfig, *, consumer_factory=None) -> No "isolation.level": "read_committed", "session.timeout.ms": config.session_timeout_ms, "max.poll.interval.ms": config.max_poll_interval_ms, + "queued.max.messages.kbytes": config.queued_max_messages_kbytes, + "fetch.max.bytes": config.fetch_max_bytes, + "max.partition.fetch.bytes": config.max_partition_fetch_bytes, "on_commit": self._on_commit, }) self._consumer.subscribe( diff --git a/qdl/transport/sqlite_spool.py b/qdl/transport/sqlite_spool.py index 5523dddc..7de21dc8 100644 --- a/qdl/transport/sqlite_spool.py +++ b/qdl/transport/sqlite_spool.py @@ -39,6 +39,8 @@ class SpoolConfig: replay_retention_seconds: int = 24 * 3600 maintenance_interval_seconds: int = 30 max_partition_records: int = 0 + retain_partition_windows: bool = False + verify_integrity_on_open: bool = True def __post_init__(self) -> None: if self.max_records <= 0 or self.max_payload_bytes <= 0: @@ -49,6 +51,10 @@ def __post_init__(self) -> None: raise ValueError("max_batch_events must be positive") if self.max_partition_records < 0: raise ValueError("max_partition_records cannot be negative") + if not isinstance(self.retain_partition_windows, bool) or ( + self.retain_partition_windows and self.max_partition_records <= 0 + ): + raise ValueError("retained partition windows require a positive record bound") if self.max_storage_bytes <= self.max_event_bytes: raise ValueError("max_storage_bytes must exceed max_event_bytes") if min( @@ -85,6 +91,18 @@ def utilization(self) -> float: ) +@dataclass(frozen=True) +class SpoolReadiness: + """Bounded, read-only cache health summary. + + `spool_state` is updated atomically alongside append and retention work. + Health checks need that durable state, not a full aggregate over `events`. + """ + + records: int + payload_bytes: int + + class SQLiteDurableSpool: """Bounded, fsync-backed migration bridge with portable logical cursors. @@ -102,7 +120,8 @@ def __init__(self, config: SpoolConfig, *, clock_ns=time.time_ns): ) self._connection.row_factory = sqlite3.Row self._initialize_schema() - self._validate_integrity() + if config.verify_integrity_on_open: + self._validate_integrity() def _initialize_schema(self) -> None: for attempt in range(4): @@ -149,7 +168,6 @@ def _migrate(self) -> None: PRIMARY KEY (stream, partition_key, logical_offset), UNIQUE (stream, event_id) ); - DROP INDEX IF EXISTS idx_qdl_spool_events_retention; CREATE INDEX IF NOT EXISTS idx_qdl_spool_events_retention ON events (committed_at_ns); @@ -183,10 +201,6 @@ def _migrate(self) -> None: payload_bytes INTEGER NOT NULL, last_maintenance_ns INTEGER NOT NULL ); - INSERT OR IGNORE INTO spool_state( - singleton, event_records, payload_bytes, last_maintenance_ns - ) - SELECT 1, COUNT(*), COALESCE(SUM(LENGTH(payload)), 0), 0 FROM events; CREATE TABLE IF NOT EXISTS cache_identity ( singleton INTEGER PRIMARY KEY CHECK (singleton = 1), @@ -195,6 +209,7 @@ def _migrate(self) -> None: ); """ ) + self._ensure_usage_state() self._connection.execute( """ INSERT OR IGNORE INTO cache_identity(singleton, cache_id, created_at_ns) @@ -203,6 +218,41 @@ def _migrate(self) -> None: (uuid.uuid4().hex, self._clock_ns()), ) + def _ensure_usage_state(self) -> None: + """Initialize legacy spool usage once without rescanning live caches.""" + with self._lock: + existing = self._connection.execute( + "SELECT 1 FROM spool_state WHERE singleton = 1" + ).fetchone() + if existing is not None: + return + self._connection.execute("BEGIN IMMEDIATE") + try: + existing = self._connection.execute( + "SELECT 1 FROM spool_state WHERE singleton = 1" + ).fetchone() + if existing is None: + records, payload_bytes = self._aggregate_event_usage_locked() + self._connection.execute( + """ + INSERT INTO spool_state( + singleton, event_records, payload_bytes, last_maintenance_ns + ) VALUES (1, ?, ?, 0) + """, + (records, payload_bytes), + ) + self._connection.execute("COMMIT") + except BaseException: + if self._connection.in_transaction: + self._connection.execute("ROLLBACK") + raise + + def _aggregate_event_usage_locked(self) -> tuple[int, int]: + row = self._connection.execute( + "SELECT COUNT(*), COALESCE(SUM(LENGTH(payload)), 0) FROM events" + ).fetchone() + return int(row[0]), int(row[1]) + @property def cache_id(self) -> str: with self._lock: @@ -398,10 +448,18 @@ def read_tail( partition_key: str, limit: int = 100, ) -> list[StoredEvent]: - """Return the newest bounded partition window in logical order.""" + """Return a bounded retained partition window in logical order. - if limit <= 0 or limit > 10_000: - raise ValueError("limit must be between 1 and 10000") + The public replay/query contract remains capped independently at + 10,000 rows. A stable runtime may retain a small, explicitly bounded + per-partition headroom for authentic late backfills; its internal + reader must be able to inspect that configured physical window before + selecting the public market-time tail. + """ + + max_tail_rows = max(10_000, self.config.max_partition_records) + if limit <= 0 or limit > max_tail_rows: + raise ValueError(f"limit must be between 1 and {max_tail_rows}") with self._lock: rows = self._connection.execute( """ @@ -535,6 +593,8 @@ def get_checkpoint( def trim_consumed(self, *, now_ns: int | None = None) -> int: """Delete only records acknowledged by every active consumer.""" + if self.config.retain_partition_windows: + return 0 effective_now = now_ns or self._clock_ns() deleted = 0 with self._lock: @@ -730,6 +790,25 @@ def stats(self) -> SpoolStats: newest_accepted_at_ns=int(row["newest"]) if row["newest"] is not None else None, ) + def readiness_summary(self) -> SpoolReadiness: + """Read the bounded, transaction-maintained usage state for health checks. + + A missing or invalid singleton is a durable-state invariant violation, + so callers deliberately fail closed rather than approximating it from + the much larger event table. + """ + with self._lock: + row = self._connection.execute( + "SELECT event_records, payload_bytes FROM spool_state WHERE singleton = 1" + ).fetchone() + if row is None: + raise PayloadCorruption("spool usage state is missing") + records = int(row["event_records"]) + payload_bytes = int(row["payload_bytes"]) + if records < 0 or payload_bytes < 0: + raise PayloadCorruption("spool usage state is invalid") + return SpoolReadiness(records=records, payload_bytes=payload_bytes) + def storage_bytes(self) -> int: return sum( path.stat().st_size @@ -804,6 +883,10 @@ def _expire_consumers_locked(self, now_ns: int) -> None: ) def _trim_aged_unowned_locked(self, now_ns: int) -> None: + # Canonical warmup windows are count-bounded: age eviction would punch + # holes into sparse bars even while their advertised window still fits. + if self.config.retain_partition_windows: + return cutoff = now_ns - self.config.replay_retention_seconds * 1_000_000_000 removed = self._connection.execute( """ diff --git a/qdl/warmup/contracts.py b/qdl/warmup/contracts.py index 02b89e6c..22f25d2d 100644 --- a/qdl/warmup/contracts.py +++ b/qdl/warmup/contracts.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from enum import StrEnum +from qdl._compat import StrEnum from typing import Any, Mapping diff --git a/qdl/warmup/executor.py b/qdl/warmup/executor.py index 8f4e1eee..e28fb7db 100644 --- a/qdl/warmup/executor.py +++ b/qdl/warmup/executor.py @@ -27,7 +27,7 @@ def __init__( @dataclass(frozen=True, slots=True) class ProviderBudgetPolicy: max_concurrency: int = 4 - requests_per_second: float = 5.0 + requests_per_second: float | None = 5.0 burst_requests: int = 5 max_attempts: int = 4 circuit_failures: int = 5 @@ -36,7 +36,7 @@ class ProviderBudgetPolicy: def __post_init__(self) -> None: if not 1 <= self.max_concurrency <= 64: raise ValueError("provider concurrency must be between 1 and 64") - if self.requests_per_second <= 0 or self.burst_requests < 1: + if (self.requests_per_second is not None and self.requests_per_second <= 0) or self.burst_requests < 1: raise ValueError("provider token budget values must be positive") if not 1 <= self.max_attempts <= 10: raise ValueError("provider attempts must be between 1 and 10") @@ -74,6 +74,11 @@ class BoundedWarmupExecutor(Generic[T, R]): """Bounded provider-fair execution with cross-request singleflight.""" DEFAULT_PROVIDER_POLICIES = { + "LOCAL_CANONICAL_CACHE": ProviderBudgetPolicy( + max_concurrency=8, + requests_per_second=None, + max_attempts=1, + ), "BINANCE": ProviderBudgetPolicy( max_concurrency=8, requests_per_second=8.0, @@ -339,6 +344,8 @@ async def _acquire_provider_token( provider: str, policy: ProviderBudgetPolicy, ) -> None: + if policy.requests_per_second is None: + return lock = self._rate_locks.setdefault(provider, asyncio.Lock()) async with lock: state = self._tokens.get(provider) diff --git a/qdl_sdk/README.md b/qdl_sdk/README.md index 0bd769a7..a387d848 100644 --- a/qdl_sdk/README.md +++ b/qdl_sdk/README.md @@ -146,7 +146,7 @@ Build the standalone artifact with: python scripts/build_qdl_sdk_release.py --output-dir dist/qdl-sdk ``` -The output contains a reproducible `qdl_sdk-2.0.0-py3-none-any.whl`, a release +The output contains a reproducible `qdl_sdk-2.0.1-py3-none-any.whl`, a release manifest with the wheel/source/generated-contract SHA-256 digests, and a CycloneDX SBOM. The wheel contains only the public SDK plus generated Protobuf contracts; it does not package `qdl.api_v2`, runtime adapters, provider code or diff --git a/qdl_sdk/__init__.py b/qdl_sdk/__init__.py index c795b5d1..c55b6478 100644 --- a/qdl_sdk/__init__.py +++ b/qdl_sdk/__init__.py @@ -1,6 +1,6 @@ """Stable Python SDK V2 with typed public models and generated transports.""" -__version__ = "2.0.0" +__version__ = "2.0.1" from qdl_sdk.client import AsyncDataLayerClient, DataLayerClientV2, WarmupStreamSession from qdl_sdk.credentials import ( diff --git a/qdl_sdk/client.py b/qdl_sdk/client.py index 8e334d75..baac6305 100644 --- a/qdl_sdk/client.py +++ b/qdl_sdk/client.py @@ -101,6 +101,32 @@ def _fixed_interval_ns(interval: str) -> int: return count * units[value[-1]] +def _allows_quiet_execution_continuity(requirement: DataRequirement, quality) -> bool: + """Admit a live but quiet non-price continuity channel. + + A stale last event is not a broken provider session. `BOOK_DELTA` is + sequence/replay evidence only, so it may be observed under an explicit + session SLA while remaining non-executable. Price-bearing feeds keep the + normal execution-eligibility gate. + """ + if ( + requirement.effective_event_recency_policy.value != "OBSERVE" + or quality.event_recency_state != "STALE" + ): + return False + if requirement.feed is Feed.TRADE: + return quality.provider_session_state in {"LIVE", "NOT_APPLICABLE"} + return ( + requirement.feed is Feed.BOOK_DELTA + and requirement.max_session_liveness_ms is not None + and quality.provider_session_state == "LIVE" + and quality.provider_session_liveness_ms is not None + and quality.provider_session_liveness_ms <= requirement.max_session_liveness_ms + and quality.complete + and not quality.gap_open + ) + + def _validate_query_payload( requirement: DataRequirement, payload: dict, *, warmup: bool ) -> SnapshotResponse | WarmupResponse: @@ -181,12 +207,7 @@ def _validate_query_payload( is_tail and requirement.consumer_grade is Grade.EXECUTION and not quality.execution_eligible - and not ( - requirement.feed is Feed.TRADE - and requirement.effective_event_recency_policy.value == "OBSERVE" - and quality.event_recency_state == "STALE" - and quality.provider_session_state in {"LIVE", "NOT_APPLICABLE"} - ) + and not _allows_quiet_execution_continuity(requirement, quality) ): raise ContinuityError( "SOURCE_NON_AUTHORITATIVE", diff --git a/qdl_sdk/projection.py b/qdl_sdk/projection.py index 2a6e2ac6..a5f36435 100644 --- a/qdl_sdk/projection.py +++ b/qdl_sdk/projection.py @@ -346,11 +346,19 @@ def market_data_view_from_stream( template: MarketDataView, requirement: DataRequirement, now_ns: int | None = None, + replay_only: bool = False, ) -> MarketDataView: """Project one SDK stream event through its authoritative query handoff. The query template owns source policy/catalog metadata. Any stream identity or source transition requires a new snapshot and fails closed here. + + ``replay_only`` is deliberately narrow: a consumer that has atomically + restored the state associated with an acknowledged durable cursor may + project an older, ordered frame only to rebuild that state. The returned + view preserves its stale/non-executable quality and the caller must obtain + a fresh strict snapshot before using a price for execution. Identity, + source, authority and gap checks remain fail-closed in both modes. """ envelope = event.event @@ -453,11 +461,12 @@ def market_data_view_from_stream( if ( event_stale and requirement.effective_event_recency_policy.value in {"BLOCK", "PAUSE"} + and not replay_only ): raise ContinuityError( "DATA_STALE", "stream event violates the requested freshness policy" ) - if session_stale: + if session_stale and not replay_only: raise ContinuityError( "DATA_STALE", "stream event transport liveness exceeds its policy" ) @@ -467,7 +476,11 @@ def market_data_view_from_stream( and template.source.source_role == "PRIMARY" and state == "LIVE" ) - if requirement.consumer_grade.value == "EXECUTION" and not authoritative: + if ( + requirement.consumer_grade.value == "EXECUTION" + and not authoritative + and not replay_only + ): code = ( "OPEN_SEQUENCE_GAP" if gap_open diff --git a/qdl_sdk/transport.py b/qdl_sdk/transport.py index b2a9ba17..09d8b1f9 100644 --- a/qdl_sdk/transport.py +++ b/qdl_sdk/transport.py @@ -1,6 +1,7 @@ from __future__ import annotations import ipaddress +import time from collections.abc import AsyncIterator, Sequence from urllib.parse import urlsplit @@ -31,6 +32,7 @@ def __init__( if client is not None and tls is not None: raise ValueError("provide either REST client or workload TLS, not both") self._owns_client = client is None + self._timeout_seconds = timeout_seconds self._credential_provider = credential_provider self._client = client or httpx.AsyncClient( base_url=self.base_url, @@ -39,9 +41,22 @@ def __init__( verify=tls.ssl_context() if tls is not None else True, ) + async def _read_request(self, method: str, path: str, **kwargs) -> httpx.Response: + # Query POSTs are read-only too. A peer can close an idle keep-alive + # socket between pool reuse and receipt of headers; retry it once. + deadline = time.monotonic() + self._timeout_seconds + request = getattr(self._client, method) + try: + return await request(path, **kwargs) + except httpx.RemoteProtocolError: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise + return await request(path, timeout=remaining, **kwargs) + async def warmup(self, requirement: DataRequirement, *, consumer_id: str) -> dict: headers = await self._headers(requirement, consumer_id) - response = await self._client.get( + response = await self._read_request("get", f"/v2/market-data/{requirement.instrument_uid}/warmup", params=requirement.query_params(), headers=headers, @@ -62,7 +77,7 @@ async def warmup_batch( if len(grades) != 1: raise ValueError("one warmup batch cannot mix consumer grades") headers = await self._identity_headers(next(iter(grades)), consumer_id) - response = await self._client.post( + response = await self._read_request("post", "/v2/market-data/warmup:batch", json={ "consumer_id": consumer_id, @@ -87,7 +102,7 @@ async def reference_batch( if len(grades) != 1: raise ValueError("one reference batch cannot mix consumer grades") headers = await self._identity_headers(next(iter(grades)), consumer_id) - response = await self._client.post( + response = await self._read_request("post", "/v2/market-data/reference:batch", json={ "consumer_id": consumer_id, @@ -102,7 +117,7 @@ async def snapshot(self, requirement: DataRequirement, *, consumer_id: str) -> d headers = await self._headers(requirement, consumer_id) params = requirement.query_params() params.pop("limit", None) - response = await self._client.get( + response = await self._read_request("get", f"/v2/market-data/{requirement.instrument_uid}/snapshot", params=params, headers=headers, @@ -124,7 +139,7 @@ async def feed_status( "end_time_ns", ): params.pop(field, None) - response = await self._client.get( + response = await self._read_request("get", f"/v2/feeds/{requirement.instrument_uid}/status", params=params, headers=headers, @@ -143,7 +158,7 @@ async def instruments( params: dict[str, str | int] = {"limit": limit} if cursor is not None: params["cursor"] = cursor - response = await self._client.get( + response = await self._read_request("get", "/v2/instruments", params=params, headers=headers ) return self._decode(response) @@ -158,7 +173,7 @@ async def instrument( if not identity.strip(): raise ValueError("instrument identity is required") headers = await self._identity_headers(consumer_grade, consumer_id) - response = await self._client.get( + response = await self._read_request("get", f"/v2/instruments/{identity}", headers=headers ) return self._decode(response) diff --git a/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs b/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs index 37fcf114..df6a214b 100644 --- a/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs +++ b/rust/qdl-kafka/src/bin/qdl-native-raw-ingestor.rs @@ -39,6 +39,7 @@ use qdl_venue_core::backpressure::DeliveryClass; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use sha2::{Digest, Sha256}; +use tokio::time::Instant; use tokio_tungstenite::connect_async; use tokio_tungstenite::tungstenite::Message; @@ -312,6 +313,14 @@ impl IngestorConfig { } } +fn authority_mode_name(mode: &AuthorityMode) -> &'static str { + match mode { + AuthorityMode::RustShadow => "RUST_SHADOW", + AuthorityMode::RustCanary => "RUST_CANARY", + AuthorityMode::RustPrimary => "RUST_PRIMARY", + } +} + fn required(name: &str) -> Result { env::var(name).map_err(|_| format!("required environment variable is missing: {name}")) } @@ -609,6 +618,7 @@ impl RawPublisher { &self, frame: RawFrameRef<'_>, stopped: &AtomicBool, + deadline: Option, ) -> Result { let record = self.record( frame.binding, @@ -629,6 +639,11 @@ impl RawPublisher { .map_err(KafkaTransportError::Configuration)?; let mut failures = 0_u32; loop { + if deadline.is_some_and(|value| Instant::now() >= value) { + return Err(KafkaTransportError::SnapshotTimeout( + "L2 raw Kafka enqueue exceeded provider renewal bound".into(), + )); + } match self.sink.enqueue(&record, &publication) { Ok(delivery) => return Ok(delivery), Err(error) @@ -647,10 +662,21 @@ impl RawPublisher { })) .unwrap_or_else(|_| "{\"event\":\"qdl_native_raw_enqueue_retry\"}".into()) ); - tokio::time::sleep(Duration::from_millis( + let delay = Duration::from_millis( backoff.delay_ms(failures, failures.min(10_000) as u16), - )) - .await; + ); + if let Some(deadline) = deadline { + tokio::select! { + _ = tokio::time::sleep(delay) => {} + _ = tokio::time::sleep_until(deadline) => { + return Err(KafkaTransportError::SnapshotTimeout( + "L2 raw Kafka enqueue exceeded provider renewal bound".into(), + )); + } + } + } else { + tokio::time::sleep(delay).await; + } } Err(error) => return Err(error), } @@ -660,8 +686,54 @@ impl RawPublisher { type RawPublishFuture = Pin> + Send>>; -fn raw_publish_future(delivery: PendingKafkaAppend) -> RawPublishFuture { - Box::pin(async move { delivery.wait().await.map(|_| ()) }) +fn raw_publish_future(delivery: PendingKafkaAppend, deadline: Option) -> RawPublishFuture { + Box::pin(async move { + if let Some(deadline) = deadline { + return tokio::time::timeout_at(deadline, delivery.wait()) + .await + .map_err(|_| { + KafkaTransportError::SnapshotTimeout( + "L2 raw Kafka delivery exceeded provider renewal bound".into(), + ) + })? + .map(|_| ()); + } + delivery.wait().await.map(|_| ()) + }) +} + +fn book_delivery_remaining_ns( + binding: &RawBinding, + received_at_ns: i64, + current_at_ns: i64, +) -> Result, KafkaTransportError> { + if binding.feed != RawFeed::Book { + return Ok(None); + } + let refresh_seconds = binding + .l2 + .as_ref() + .and_then(|l2| l2.snapshot_refresh_seconds) + .ok_or_else(|| { + KafkaTransportError::Configuration( + "BOOK binding is missing validated snapshot renewal configuration".into(), + ) + })?; + let max_age_ns = refresh_seconds + .checked_mul(1_000_000_000) + .ok_or_else(|| KafkaTransportError::Configuration("BOOK renewal bound overflow".into()))?; + let max_age_ns = i64::try_from(max_age_ns) + .map_err(|_| KafkaTransportError::Configuration("BOOK renewal bound exceeds i64".into()))?; + let elapsed_ns = current_at_ns.saturating_sub(received_at_ns).max(0); + let remaining_ns = max_age_ns.saturating_sub(elapsed_ns); + if remaining_ns <= 0 { + return Err(KafkaTransportError::SnapshotTimeout( + "L2 raw frame exceeded provider renewal bound before Kafka delivery".into(), + )); + } + Ok(Some(u64::try_from(remaining_ns).map_err(|_| { + KafkaTransportError::Configuration("BOOK delivery remaining time is invalid".into()) + })?)) } #[derive(Clone, Debug)] @@ -912,6 +984,20 @@ async fn enqueue_lossless_frame( if !reserve(accepted, max_events).await { return Ok(false); } + let remaining_ns = match book_delivery_remaining_ns( + &frame.binding, + frame.received_at_ns, + now_ns().map_err(|error| KafkaTransportError::Configuration(error.to_string()))?, + ) { + Ok(value) => value, + Err(error) => { + if max_events > 0 { + accepted.fetch_sub(1, Ordering::AcqRel); + } + return Err(error); + } + }; + let deadline = remaining_ns.map(|value| Instant::now() + Duration::from_nanos(value)); let delivery = publisher .enqueue_with_retry( RawFrameRef { @@ -923,11 +1009,12 @@ async fn enqueue_lossless_frame( transport_protocol: frame.transport_protocol, }, stopped, + deadline, ) .await; match delivery { Ok(delivery) => { - inflight.push(raw_publish_future(delivery)); + inflight.push(raw_publish_future(delivery, deadline)); Ok(true) } Err(error) => { @@ -2003,7 +2090,7 @@ async fn main() -> Result<(), Box> { serde_json::to_string(&json!({ "event": "qdl_native_raw_ingestor_started", "runtime": format!("{:?}", config.runtime).to_ascii_uppercase(), - "authority": "RUST_SHADOW", + "authority": authority_mode_name(&config.authority.mode), "bindings": config.bindings.len(), "latest_state_flush_ms": config.latest_state_flush_ms, "production_public_writes": 0, @@ -2088,10 +2175,11 @@ async fn main() -> Result<(), Box> { #[cfg(test)] mod tests { use super::{ - book_snapshot_renewal_period, next_connection_generation, partition_binance_bindings, - partition_bindings, partition_okx_bindings, pending_binance_frame, pending_okx_frame, - DeliveryClass, LatestStateBuffer, PendingRawFrame, ProviderRuntime, RawBinding, RawFeed, - SessionLivenessWriter, + authority_mode_name, book_delivery_remaining_ns, book_snapshot_renewal_period, + next_connection_generation, partition_binance_bindings, partition_bindings, + partition_okx_bindings, pending_binance_frame, pending_okx_frame, AuthorityMode, + DeliveryClass, KafkaTransportError, LatestStateBuffer, PendingRawFrame, ProviderRuntime, + RawBinding, RawFeed, SessionLivenessWriter, }; use std::collections::HashMap; use std::fs; @@ -2132,6 +2220,47 @@ mod tests { } } + #[test] + fn book_delivery_bound_is_exact_and_never_applies_to_non_book_frames() { + let mut book = binding(RawFeed::Book, DeliveryClass::Lossless); + book.l2 = Some(super::RawL2Config { + provider_protocol: "BINANCE_DIFF_DEPTH".into(), + depth_per_side: 100, + rest_snapshot_url: Some("https://example.test/depth".into()), + snapshot_refresh_seconds: Some(30), + }); + assert_eq!( + book_delivery_remaining_ns(&book, 1_000, 29_000_001_000).unwrap(), + Some(1_000_000_000) + ); + assert!(matches!( + book_delivery_remaining_ns(&book, 1_000, 30_000_001_001), + Err(KafkaTransportError::SnapshotTimeout(_)) + )); + + let trade = binding(RawFeed::Trade, DeliveryClass::Lossless); + assert_eq!( + book_delivery_remaining_ns(&trade, 1, i64::MAX).unwrap(), + None + ); + } + + #[test] + fn startup_authority_name_matches_fenced_mode() { + assert_eq!( + authority_mode_name(&AuthorityMode::RustShadow), + "RUST_SHADOW" + ); + assert_eq!( + authority_mode_name(&AuthorityMode::RustCanary), + "RUST_CANARY" + ); + assert_eq!( + authority_mode_name(&AuthorityMode::RustPrimary), + "RUST_PRIMARY" + ); + } + #[test] fn provider_bindings_are_sharded_without_truncation() { let values: Vec = (0..205) diff --git a/rust/qdl-kafka/src/bin/qdl-realtime-core.rs b/rust/qdl-kafka/src/bin/qdl-realtime-core.rs index b29d8cb0..0ae95435 100644 --- a/rust/qdl-kafka/src/bin/qdl-realtime-core.rs +++ b/rust/qdl-kafka/src/bin/qdl-realtime-core.rs @@ -19,6 +19,8 @@ use qdl_venue_core::authority::{AuthorityMode, AuthorityRecord, PublicationConte use serde::Deserialize; use serde_json::json; +const MAX_IN_GENERATION_RECEIVE_RETRIES: u32 = 3; + #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct RuntimeConfig { @@ -106,6 +108,10 @@ fn retryable_runtime_error(error: &RuntimeError) -> bool { .is_some_and(|value| should_retry_transport(value.retry_class())) } +fn should_retry_receive_in_generation(error: &KafkaTransportError, failures: u32) -> bool { + should_retry_transport(error.retry_class()) && failures < MAX_IN_GENERATION_RECEIVE_RETRIES +} + fn approved_subscription_scope(config: &RuntimeConfig) -> HashSet { config .core @@ -164,7 +170,11 @@ fn authority_mode_name(mode: AuthorityMode) -> &'static str { } } -async fn run_generation(config: &RuntimeConfig, generation: u64) -> Result<(), RuntimeError> { +async fn run_generation( + config: &RuntimeConfig, + generation: u64, + backoff: &BackoffPolicy, +) -> Result<(), RuntimeError> { let mut core = RealtimeCore::new(config.core.clone())?; let approved_subscriptions = approved_subscription_scope(config); let (canonical_target, quarantine_target) = output_targets(config.authority.mode)?; @@ -200,15 +210,47 @@ async fn run_generation(config: &RuntimeConfig, generation: u64) -> Result<(), R let mut ignored_out_of_scope = 0_u64; let mut scope_quarantines = 0_u64; let mut batches = 0_u64; + let mut receive_failures = 0_u32; let shutdown = shutdown_signal(); tokio::pin!(shutdown); let stop_reason = 'service: loop { if config.max_events > 0 && processed >= config.max_events { break 'service "MAX_EVENTS"; } - let first = tokio::select! { - result = bridge.next() => result?, - result = &mut shutdown => break 'service result?.as_str(), + let first = loop { + let received = tokio::select! { + result = bridge.next() => result, + result = &mut shutdown => break 'service result?.as_str(), + }; + match received { + Ok(input) => { + receive_failures = 0; + break input; + } + Err(error) if should_retry_receive_in_generation(&error, receive_failures) => { + receive_failures = receive_failures.saturating_add(1); + eprintln!( + "{}", + serde_json::to_string(&json!({ + "event": "qdl_realtime_core_receive_retry", + "generation": generation, + "attempt": receive_failures, + "max_attempts": MAX_IN_GENERATION_RECEIVE_RETRIES, + "error": error.to_string(), + }))? + ); + tokio::select! { + _ = tokio::time::sleep(Duration::from_millis( + backoff.delay_ms( + receive_failures.saturating_sub(1), + receive_failures as u16, + ), + )) => {} + result = &mut shutdown => break 'service result?.as_str(), + } + } + Err(error) => return Err(error.into()), + } }; let mut inputs = vec![first]; while inputs.len() < config.batch_size @@ -217,7 +259,24 @@ async fn run_generation(config: &RuntimeConfig, generation: u64) -> Result<(), R match tokio::time::timeout(Duration::from_millis(config.batch_wait_ms), bridge.next()) .await { - Ok(Ok(input)) => inputs.push(input), + Ok(Ok(input)) => { + receive_failures = 0; + inputs.push(input); + } + Ok(Err(error)) if should_retry_receive_in_generation(&error, receive_failures) => { + receive_failures = receive_failures.saturating_add(1); + eprintln!( + "{}", + serde_json::to_string(&json!({ + "event": "qdl_realtime_core_receive_deferred", + "generation": generation, + "attempt": receive_failures, + "max_attempts": MAX_IN_GENERATION_RECEIVE_RETRIES, + "error": error.to_string(), + }))? + ); + break; + } Ok(Err(error)) => return Err(error.into()), Err(_) => break, } @@ -363,7 +422,7 @@ async fn main() -> Result<(), RuntimeError> { let mut failures = 0_u32; let result = loop { generation = generation.saturating_add(1); - match run_generation(&config, generation).await { + match run_generation(&config, generation, &backoff).await { Ok(()) => break Ok(()), Err(error) if retryable_runtime_error(&error) => { failures = failures.saturating_add(1); @@ -401,6 +460,23 @@ mod tests { assert!(!should_retry_transport(RetryClass::NonRetryable)); } + #[test] + fn retryable_receive_errors_preserve_generation_only_within_bound() { + let retryable = KafkaTransportError::SnapshotTimeout("transient receive".into()); + assert!(should_retry_receive_in_generation(&retryable, 0)); + assert!(should_retry_receive_in_generation( + &retryable, + MAX_IN_GENERATION_RECEIVE_RETRIES - 1, + )); + assert!(!should_retry_receive_in_generation( + &retryable, + MAX_IN_GENERATION_RECEIVE_RETRIES, + )); + + let fatal = KafkaTransportError::Configuration("bad config".into()); + assert!(!should_retry_receive_in_generation(&fatal, 0)); + } + #[test] fn shared_raw_scope_accepts_only_exact_subscription_ids() { let approved_subscriptions = HashSet::from(["dnse-vn30f1m-trade-stable-001".to_owned()]); diff --git a/rust/qdl-realtime-core/src/lib.rs b/rust/qdl-realtime-core/src/lib.rs index c1f511eb..e8e6e44e 100644 --- a/rust/qdl-realtime-core/src/lib.rs +++ b/rust/qdl-realtime-core/src/lib.rs @@ -38,13 +38,31 @@ pub enum L2ProviderProtocol { pub struct L2Binding { pub provider_protocol: L2ProviderProtocol, pub depth_per_side: usize, + // Provider bootstrap/renewal remains deliberately slower than the + // canonical verified-view materialization cadence. pub snapshot_refresh_seconds: u64, + #[serde(default)] + pub materialized_snapshot_interval_ms: Option, } impl L2Binding { + fn materialized_snapshot_interval_ms(&self) -> u64 { + self.materialized_snapshot_interval_ms + .unwrap_or_else(|| self.snapshot_refresh_seconds.saturating_mul(1_000)) + } + + fn raw_receipt_is_expired(&self, received_at_ns: i64, processing_at_ns: i64) -> bool { + let max_age_ns = self.snapshot_refresh_seconds.saturating_mul(1_000_000_000); + let max_age_ns = i64::try_from(max_age_ns).unwrap_or(i64::MAX); + processing_at_ns.saturating_sub(received_at_ns) > max_age_ns + } + fn validate(&self, binding: &CoreBinding) -> Result<(), CoreError> { + let materialized_snapshot_interval_ms = self.materialized_snapshot_interval_ms(); if !(1..=10_000).contains(&self.depth_per_side) || !(5..=300).contains(&self.snapshot_refresh_seconds) + || !(100..=self.snapshot_refresh_seconds.saturating_mul(1_000)) + .contains(&materialized_snapshot_interval_ms) || binding.sequence_policy != SequencePolicy::Contiguous { return Err(CoreError::Configuration( @@ -378,7 +396,7 @@ impl RealtimeCore { &binding, raw, payload, - materialized_at_ns, + processing_at_ns, transport_offset, )); } @@ -571,12 +589,24 @@ impl RealtimeCore { binding: &CoreBinding, raw: RawProviderEnvelope, payload: Value, - materialized_at_ns: i64, + processing_at_ns: i64, transport_offset: Option, ) -> ProcessBatch { + let materialized_at_ns = raw.received_at_ns; let Some(l2) = binding.l2.as_ref() else { unreachable!("process_l2 only accepts an L2 binding") }; + if l2.raw_receipt_is_expired(raw.received_at_ns, processing_at_ns) { + if let Some(adapter) = self.l2_adapters.get_mut(binding_key) { + adapter.request_resync(raw.connection_generation); + } + return self.quarantine( + &raw, + QuarantineReason::SemanticInvalid, + "L2 raw receipt exceeded provider renewal bound; resync required", + processing_at_ns, + ); + } let transport = match TransportProtocol::try_from(raw.transport_protocol) { Ok(value) => value, Err(_) => { @@ -769,9 +799,7 @@ impl RealtimeCore { return None; } let observed_at_ms = l2_observed_at_ms(transition, raw)?; - let refresh_ms = i64::try_from(l2.snapshot_refresh_seconds) - .ok()? - .saturating_mul(1_000); + let refresh_ms = i64::try_from(l2.materialized_snapshot_interval_ms()).ok()?; let due = self .l2_snapshot_observed_at_ms .get(binding_key) @@ -1103,6 +1131,7 @@ mod tests { provider_protocol: L2ProviderProtocol::BinanceDiffDepth, depth_per_side: 100, snapshot_refresh_seconds: 30, + materialized_snapshot_interval_ms: None, }); result } @@ -1124,6 +1153,7 @@ mod tests { provider_protocol: L2ProviderProtocol::OkxPublicBooks, depth_per_side: 100, snapshot_refresh_seconds: 30, + materialized_snapshot_interval_ms: None, }); result } @@ -1271,6 +1301,71 @@ mod tests { assert_eq!(snapshot.partition_sequence, delta.partition_sequence + 1); } + #[test] + fn binance_l2_hot_materialization_is_independent_of_provider_refresh() { + let mut binding = binance_book_binding(); + let l2 = binding.l2.as_mut().expect("L2 binding"); + l2.materialized_snapshot_interval_ms = Some(1_000); + assert_eq!(l2.snapshot_refresh_seconds, 30); + let mut core = core(binding.clone(), true); + + let buffered = core + .process( + raw( + &binding, + br#"{"s":"BTCUSDT","U":101,"u":101,"E":1000,"b":[["60000","1"]],"a":[]}"#, + 1, + ), + 11, + ) + .unwrap(); + assert!(buffered.canonical.is_empty()); + + let initial = core + .process( + with_transport( + raw( + &binding, + br#"{"lastUpdateId":100,"E":1000,"bids":[["60000","1"]],"asks":[["60001","1"]]}"#, + 1, + ), + TransportProtocol::Http, + ), + 12, + ) + .unwrap(); + assert_eq!(initial.canonical.len(), 1); + + let early = core + .process( + raw( + &binding, + br#"{"s":"BTCUSDT","U":102,"u":102,"pu":101,"E":1999,"b":[["60000","2"]],"a":[]}"#, + 1, + ), + 13, + ) + .unwrap(); + assert_eq!(early.canonical.len(), 1); + + let due = core + .process( + raw( + &binding, + br#"{"s":"BTCUSDT","U":103,"u":103,"pu":102,"E":2000,"b":[["60000","3"]],"a":[]}"#, + 1, + ), + 14, + ) + .unwrap(); + assert_eq!(due.canonical.len(), 2); + let snapshot = EventEnvelope::decode(due.canonical[1].payload.as_slice()).unwrap(); + assert!(matches!( + snapshot.payload, + Some(event_envelope::Payload::BookSnapshot(_)) + )); + } + #[test] fn binance_l2_due_keepalive_materializes_current_verified_view() { let binding = binance_book_binding(); @@ -1369,6 +1464,50 @@ mod tests { assert_eq!(evidence.reason, QuarantineReason::SequenceGap as i32); } + #[test] + fn stale_l2_raw_is_quarantined_without_redating_and_fresh_snapshot_recovers() { + let binding = binance_book_binding(); + let mut core = core(binding.clone(), true); + let stale = with_transport( + raw( + &binding, + br#"{"lastUpdateId":100,"bids":[["60000","1"]],"asks":[["60001","1"]]}"#, + 1, + ), + TransportProtocol::Http, + ); + let processing_at_ns = stale.received_at_ns + 30_000_000_001; + let rejected = core.process(stale, processing_at_ns).unwrap(); + assert!(rejected.canonical.is_empty()); + assert_eq!(rejected.quarantines.len(), 1); + let evidence = + QuarantineRecord::decode(rejected.quarantines[0].payload.as_slice()).unwrap(); + assert_eq!(evidence.reason, QuarantineReason::SemanticInvalid as i32); + assert_eq!(evidence.quarantined_at_ns, processing_at_ns); + assert!(evidence.safe_summary.contains("renewal bound")); + + let mut fresh_delta = raw( + &binding, + br#"{"s":"BTCUSDT","U":99,"u":101,"pu":98,"E":1001,"b":[["60000","2"]],"a":[["60001","1"]]}"#, + 2, + ); + fresh_delta.received_at_ns = processing_at_ns; + let buffered = core.process(fresh_delta, processing_at_ns).unwrap(); + assert!(buffered.canonical.is_empty()); + + let mut fresh_snapshot = with_transport( + raw( + &binding, + br#"{"lastUpdateId":100,"bids":[["60000","2"]],"asks":[["60001","1"]]}"#, + 2, + ), + TransportProtocol::Http, + ); + fresh_snapshot.received_at_ns = processing_at_ns; + let recovered = core.process(fresh_snapshot, processing_at_ns).unwrap(); + assert_eq!(recovered.canonical.len(), 1); + } + #[test] fn okx_l2_snapshot_then_update_emit_distinct_public_events() { let binding = okx_book_binding(); diff --git a/rust/qdl-venue-core/src/demand.rs b/rust/qdl-venue-core/src/demand.rs index c608d8e6..cf2acdb2 100644 --- a/rust/qdl-venue-core/src/demand.rs +++ b/rust/qdl-venue-core/src/demand.rs @@ -105,6 +105,10 @@ pub struct ReconcileAction { pub reason: String, } +// The sealed V2 final-BAR contract permits a weekly bar to remain governed by +// its 21-day freshness window. Individual demand policy stays authoritative. +const MAX_REQUIREMENT_FRESHNESS_MS: u64 = 21 * 86_400_000; + pub fn validate_requirement(requirement: &DataRequirement) -> Result<(), String> { let purpose = DemandPurpose::try_from(requirement.purpose).map_err(|_| "demand purpose is invalid")?; @@ -157,7 +161,7 @@ pub fn validate_requirement(requirement: &DataRequirement) -> Result<(), String> if (!universe.expected_universe_sha256.is_empty() && universe.expected_universe_sha256.len() != 32) || requirement.warmup_limit > 100_000 - || requirement.max_freshness_ms > 86_400_000 + || requirement.max_freshness_ms > MAX_REQUIREMENT_FRESHNESS_MS || requirement.priority > 1_000 || requirement.ttl_seconds < 30 || requirement.ttl_seconds > 3_600 @@ -402,7 +406,7 @@ pub fn transition_allowed(previous: DemandState, next: DemandState) -> bool { mod tests { use super::{ plan_shards, reconcile, transition_allowed, validate_requirement, DemandSubscription, - ReconcileActionKind, + ReconcileActionKind, MAX_REQUIREMENT_FRESHNESS_MS, }; use qdl_contracts::qdl::demand::v1::{ universe_selector, warmup_specification, DataRequirement, DemandPurpose, DemandState, @@ -477,7 +481,9 @@ mod tests { .expected_universe_sha256 = vec![0; 31]; assert!(validate_requirement(&invalid_digest).is_err()); let mut invalid_freshness = requirement(); - invalid_freshness.max_freshness_ms = 86_400_001; + invalid_freshness.max_freshness_ms = MAX_REQUIREMENT_FRESHNESS_MS; + assert!(validate_requirement(&invalid_freshness).is_ok()); + invalid_freshness.max_freshness_ms = MAX_REQUIREMENT_FRESHNESS_MS + 1; assert!(validate_requirement(&invalid_freshness).is_err()); } diff --git a/scripts/build_qdl_sdk_release.py b/scripts/build_qdl_sdk_release.py index 2b3abd57..9109b4e7 100755 --- a/scripts/build_qdl_sdk_release.py +++ b/scripts/build_qdl_sdk_release.py @@ -15,7 +15,7 @@ ROOT = Path(__file__).resolve().parents[1] NAME = "qdl-sdk" NORMALIZED_NAME = "qdl_sdk" -VERSION = "2.0.0" +VERSION = "2.0.1" DIST_INFO = f"{NORMALIZED_NAME}-{VERSION}.dist-info" DEPENDENCIES = ( "grpcio>=1.70.0,<2.0.0", diff --git a/scripts/compile_alpha_deployment_bindings.py b/scripts/compile_alpha_deployment_bindings.py new file mode 100644 index 00000000..de1de631 --- /dev/null +++ b/scripts/compile_alpha_deployment_bindings.py @@ -0,0 +1,597 @@ +"""Compile secret-free V2 route bindings from alpha deployment configuration. + +The Execution Alpha repository owns the portable +``execution-alpha.data-requirements.v1`` inventory. This compiler resolves +that inventory against the Data Layer's existing catalog and reference/L2 +entitlements, then emits the *existing* ``qdl.v2.consumer-route-binding.v1`` +contract which Trading System and alpha SDK already parse. + +It is control-plane/source tooling only: it never opens a provider connection, +starts a subscriber, writes a runtime bundle, or changes an existing consumer. +An unsupported or mismatched deployment is represented by one typed BLOCKED +result. The compiler never drops one required route and returns a partial +binding. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import re +from typing import Any, Mapping + +import yaml + +from qdl.consumer.universal_release import ConsumerRouteBinding + + +ROOT = Path(__file__).resolve().parents[1] +INVENTORY_SCHEMA = "execution-alpha.data-requirements.v1" +REPORT_SCHEMA = "qdl.v2.alpha-deployment-binding-compilation.v1" +CONTRACT_VERSION = "2.0.0" +_V2_VENUES = frozenset({"BINANCE", "OKX"}) +_REFERENCE_FEEDS = frozenset( + { + "FUNDING_RATE", + "OPEN_INTEREST", + "LONG_SHORT_RATIO", + "TAKER_FLOW", + "MARK_INDEX_PRICE", + "CONTRACT_METADATA", + "BASIS", + } +) +_METRIC_INTERVAL_FEEDS = frozenset({"OPEN_INTEREST", "LONG_SHORT_RATIO", "TAKER_FLOW", "BASIS"}) +_PROFILE_CLASSES = { + "directional_bar": "SINGLE_SYMBOL_ALPHA", + "multi_symbol_bar": "PORTFOLIO_MULTI_SYMBOL", + "bracket_context": "SINGLE_SYMBOL_ALPHA", + "grid_l2": "GRID_REACTIVE_BRACKET", + "basis_reference": "BASIS_ARB", +} +_SAFE_FILE_COMPONENT = re.compile(r"[^a-zA-Z0-9_.-]+") + + +class DeploymentBindingError(ValueError): + """Raised when an inventory or compiler source artifact is malformed.""" + + +def _canonical_bytes(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def _digest(value: object) -> str: + return hashlib.sha256(_canonical_bytes(value)).hexdigest() + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _sha256_file(path: Path) -> str: + return _sha256_bytes(path.read_bytes()) + + +def _text(value: object, field: str) -> str: + result = str(value or "").strip() + if not result: + raise DeploymentBindingError(f"{field} is required") + return result + + +def _sha256(value: object, field: str) -> str: + result = _text(value, field).lower() + if len(result) != 64 or any(character not in "0123456789abcdef" for character in result): + raise DeploymentBindingError(f"{field} must be a SHA-256 digest") + return result + + +def _positive_int(value: object, field: str, *, maximum: int | None = None) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise DeploymentBindingError(f"{field} must be a positive integer") + if maximum is not None and value > maximum: + raise DeploymentBindingError(f"{field} must not exceed {maximum}") + return value + + +def _bool(value: object, field: str) -> bool: + if not isinstance(value, bool): + raise DeploymentBindingError(f"{field} must be boolean") + return value + + +def _mapping(value: object, field: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise DeploymentBindingError(f"{field} must be a mapping") + return value + + +def _load_mapping(path: Path) -> dict[str, Any]: + try: + value = yaml.safe_load(path.read_text(encoding="utf-8")) + except FileNotFoundError as error: + raise DeploymentBindingError(f"source file is missing: {path}") from error + if not isinstance(value, dict): + raise DeploymentBindingError(f"{path} must be a mapping") + return value + + +def _load_json_mapping(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as error: + raise DeploymentBindingError(f"inventory is missing: {path}") from error + except json.JSONDecodeError as error: + raise DeploymentBindingError("inventory is not valid JSON") from error + if not isinstance(value, dict): + raise DeploymentBindingError("inventory must be a JSON object") + return value + + +def _route_key( + *, + instrument_uid: str, + feed: str, + interval: str | None, + source_policy_id: str, +) -> tuple[str, str, str | None, str]: + return ( + instrument_uid, + feed.upper(), + interval.lower() if interval is not None else None, + source_policy_id, + ) + + +def _inventory_digest(inventory: Mapping[str, Any]) -> str: + raw = dict(inventory) + reported = _sha256(raw.pop("inventory_sha256", None), "inventory_sha256") + if _digest(raw) != reported: + raise DeploymentBindingError("inventory checksum differs") + return reported + + +def _validate_inventory(inventory: Mapping[str, Any]) -> tuple[Mapping[str, Any], ...]: + expected = { + "schema", "revision", "registry_path", "registry_sha256", "deployments", "inventory_sha256" + } + if set(inventory) != expected or inventory.get("schema") != INVENTORY_SCHEMA: + raise DeploymentBindingError("inventory schema or fields are invalid") + _positive_int(inventory.get("revision"), "inventory revision") + _sha256(inventory.get("registry_sha256"), "inventory registry_sha256") + _inventory_digest(inventory) + deployments = inventory.get("deployments") + if not isinstance(deployments, list) or not deployments: + raise DeploymentBindingError("inventory deployments are required") + result: list[Mapping[str, Any]] = [] + seen: set[str] = set() + for item in deployments: + deployment = _mapping(item, "inventory deployment") + deployment_id = _text(deployment.get("deployment_id"), "deployment_id") + if deployment_id in seen: + raise DeploymentBindingError("inventory deployment_id is duplicated") + seen.add(deployment_id) + status = _text(deployment.get("status"), "deployment status") + if status not in {"DECLARED", "DECLARED_NO_ORDER_PROBE", "BLOCKED"}: + raise DeploymentBindingError("inventory deployment status is invalid") + result.append(deployment) + return tuple(sorted(result, key=lambda item: str(item["deployment_id"]))) + + +def _catalog_indexes( + catalog: Mapping[str, Any], +) -> tuple[ + dict[tuple[str, str, str, str], Mapping[str, Any]], + dict[tuple[str, str, str | None, str], Mapping[str, Any]], +]: + if catalog.get("schema") != "qdl.v2.stable-source-bindings.v1": + raise DeploymentBindingError("stable source catalog schema is unsupported") + records: dict[tuple[str, str, str, str], Mapping[str, Any]] = {} + by_uid: dict[str, Mapping[str, Any]] = {} + for item in catalog.get("instruments", []): + instrument = _mapping(item, "catalog instrument") + identity = tuple( + _text(instrument.get(field), f"catalog instrument {field}").upper() + for field in ("venue", "market", "product_type", "native_symbol") + ) + uid = _text(instrument.get("instrument_uid"), "catalog instrument_uid") + if identity in records or uid in by_uid: + raise DeploymentBindingError("catalog instrument identity is duplicated") + records[identity] = instrument + by_uid[uid] = instrument + bindings: dict[tuple[str, str, str | None, str], Mapping[str, Any]] = {} + for item in catalog.get("bindings", []): + binding = _mapping(item, "catalog binding") + source = _mapping(binding.get("source"), "catalog binding source") + key = _route_key( + instrument_uid=_text(binding.get("instrument_uid"), "catalog binding instrument_uid"), + feed=_text(binding.get("feed"), "catalog binding feed"), + interval=(str(binding["interval"]) if binding.get("interval") is not None else None), + source_policy_id=_text(source.get("source_policy_id"), "catalog source policy"), + ) + if key in bindings: + raise DeploymentBindingError("catalog binding identity is duplicated") + if key[0] not in by_uid: + raise DeploymentBindingError("catalog binding references undeclared instrument") + bindings[key] = binding + return records, bindings + + +def _reference_keys(reference_manifest: Mapping[str, Any]) -> set[tuple[str, str, str | None, str]]: + spec = _mapping(reference_manifest.get("spec"), "reference manifest spec") + requirements = spec.get("requirements") + if not isinstance(requirements, list): + raise DeploymentBindingError("reference manifest requirements are invalid") + keys: set[tuple[str, str, str | None, str]] = set() + for item in requirements: + row = _mapping(item, "reference requirement") + keys.add(_route_key( + instrument_uid=_text(row.get("instrument_uid"), "reference instrument_uid"), + feed=_text(row.get("feed"), "reference feed"), + interval=(str(row["interval"]) if row.get("interval") is not None else None), + source_policy_id=_text(row.get("source_policy_id"), "reference source policy"), + )) + return keys + + +def _consumer_id(deployment: Mapping[str, Any]) -> str: + alpha_id = _text(deployment.get("alpha_id"), "deployment alpha_id") + status = _text(deployment.get("status"), "deployment status") + suffix = "okx.no-order" if status == "DECLARED_NO_ORDER_PROBE" else "binance.paper" + return f"alpha.{alpha_id}.{suffix}" + + +def _consumer_class(deployment: Mapping[str, Any]) -> str: + profile = _text(deployment.get("profile"), "deployment profile") + try: + return _PROFILE_CLASSES[profile] + except KeyError as error: + raise DeploymentBindingError(f"deployment profile is not classified: {profile}") from error + + +def _route_product( + *, + deployment: Mapping[str, Any], + route: Mapping[str, Any], + instrument: Mapping[str, Any], + consumer_id: str, + consumer_class: str, +) -> dict[str, Any]: + feed = _text(route.get("feed"), "route feed").upper() + interval = str(route["interval"]).strip().lower() if route.get("interval") is not None else None + if feed == "BAR" and not interval: + raise DeploymentBindingError("BAR interval identity is invalid") + if feed not in {"BAR", *_METRIC_INTERVAL_FEEDS} and interval is not None: + raise DeploymentBindingError("non-BAR route declares an interval") + fallback = _text(route.get("fallback"), "route fallback").upper() + if fallback not in {"V1", "BLOCKED"}: + raise DeploymentBindingError("route fallback is invalid") + venue = _text(route.get("venue"), "route venue").upper() + market = _text(route.get("market"), "route market").upper() + product_type = _text(route.get("product_type"), "route product_type").upper() + if fallback == "V1" and (venue, market, product_type, feed, interval) != ( + "BINANCE", "USDM", "PERPETUAL", "TRADE", None + ): + raise DeploymentBindingError("V1 fallback is allowed only for native Binance USD-M TRADE") + fallback_rule_id = route.get("fallback_rule_id") if fallback == "V1" else None + blocked_reason = route.get("blocked_reason") if fallback == "BLOCKED" else None + if fallback == "V1" and not _text(fallback_rule_id, "route fallback_rule_id"): + raise DeploymentBindingError("V1 route requires fallback_rule_id") + if fallback == "BLOCKED" and not _text(blocked_reason, "route blocked_reason"): + raise DeploymentBindingError("blocked route requires blocked_reason") + identity = { + "deployment_id": _text(deployment.get("deployment_id"), "deployment_id"), + "venue": venue, + "market": market, + "product_type": product_type, + "native_symbol": _text(route.get("native_symbol"), "route native_symbol").upper(), + "feed": feed, + "interval": interval, + "source_policy_id": _text(route.get("source_policy_id"), "route source_policy_id"), + } + return { + "consumer_id": consumer_id, + "consumer_class": consumer_class, + "requirement_id": _digest(identity), + "instrument_uid": _text(instrument.get("instrument_uid"), "instrument_uid"), + "instrument_id": _text(instrument.get("instrument_id"), "instrument_id"), + "venue": venue, + "market": market, + "product_type": product_type, + "native_symbol": identity["native_symbol"], + "feed": feed, + "interval": interval, + "source_policy_id": identity["source_policy_id"], + "provider_plane": _text(route.get("provider_plane"), "route provider_plane").upper(), + "max_freshness_ms": _positive_int(route.get("max_freshness_ms"), "route max_freshness_ms"), + "require_final_bars": _bool(route.get("require_final_bars", False), "route require_final_bars"), + "require_live": _bool(route.get("require_live", False), "route require_live"), + # Alpha context remains advisory. Risk owns execution-grade rereads. + "execution_grade": False, + "route": "V2_PRIMARY", + "fallback": fallback, + "fallback_rule_id": str(fallback_rule_id) if fallback_rule_id is not None else None, + "blocked_reason": str(blocked_reason) if blocked_reason is not None else None, + "gap_policy": "BLOCK", + } + + +def _missing_reason( + *, + route: Mapping[str, Any], + instrument: Mapping[str, Any], + bindings: Mapping[tuple[str, str, str | None, str], Mapping[str, Any]], + reference_keys: set[tuple[str, str, str | None, str]], +) -> str | None: + feed = _text(route.get("feed"), "route feed").upper() + interval = str(route["interval"]).strip().lower() if route.get("interval") is not None else None + key = _route_key( + instrument_uid=_text(instrument.get("instrument_uid"), "instrument_uid"), + feed=feed, + interval=interval, + source_policy_id=_text(route.get("source_policy_id"), "route source_policy_id"), + ) + binding = bindings.get(key) + if binding is None and key not in reference_keys: + return f"CAPABILITY_UNAVAILABLE:{feed}:{key[3]}" + if feed == "BAR" and bool(route.get("require_final_bars", False)): + quality = _mapping(binding.get("quality"), "catalog BAR quality") if binding is not None else None + if quality is None or quality.get("require_final_bar") is not True: + return "FINAL_BAR_UNAVAILABLE" + if feed not in _REFERENCE_FEEDS and binding is None: + return f"CATALOG_BINDING_UNAVAILABLE:{feed}:{key[3]}" + return None + + +def _unavailable_optional(route: Mapping[str, Any], reason: str) -> dict[str, Any]: + """Preserve a missing optional route as explicit absence, never as zero.""" + + return { + "venue": _text(route.get("venue"), "route venue").upper(), + "market": _text(route.get("market"), "route market").upper(), + "product_type": _text(route.get("product_type"), "route product_type").upper(), + "native_symbol": _text(route.get("native_symbol"), "route native_symbol").upper(), + "feed": _text(route.get("feed"), "route feed").upper(), + "interval": str(route["interval"]).strip().lower() if route.get("interval") is not None else None, + "source_policy_id": _text(route.get("source_policy_id"), "route source_policy_id"), + "reason": reason, + } + + +def _blocked_result(deployment: Mapping[str, Any], reason: str) -> dict[str, Any]: + return { + "deployment_id": _text(deployment.get("deployment_id"), "deployment_id"), + "alpha_id": str(deployment.get("alpha_id") or ""), + "status": "BLOCKED", + "reason": reason, + "optional_unavailable": [], + "binding": None, + } + + +def compile_inventory( + *, + inventory: Mapping[str, Any], + catalog: Mapping[str, Any], + reference_manifest: Mapping[str, Any], + release_routing: Mapping[str, Any], + release_routing_sha256: str, + policy: Mapping[str, Any], + policy_sha256: str, + catalog_sha256: str, + reference_manifest_sha256: str, +) -> dict[str, Any]: + """Resolve one portable alpha inventory into standard sealed bindings.""" + + deployments = _validate_inventory(inventory) + records, bindings = _catalog_indexes(catalog) + reference_keys = _reference_keys(reference_manifest) + if release_routing.get("schema") != "qdl.v2.stable-release-routing.v1": + raise DeploymentBindingError("release routing schema is unsupported") + if policy.get("schema") != "qdl.v2.universal-release-policy.v1": + raise DeploymentBindingError("release policy schema is unsupported") + release_revision = _positive_int(release_routing.get("revision"), "release routing revision") + capability = _mapping(release_routing.get("capability_matrix"), "release capability matrix") + rollback = _mapping(policy.get("v1_rollback"), "release rollback") + required_rollback = {"release_tag", "source_commit", "image_reference", "manifest_revision"} + if set(rollback) != required_rollback: + raise DeploymentBindingError("release rollback fields are invalid") + _sha256(release_routing_sha256, "release_routing_sha256") + _sha256(policy_sha256, "policy_sha256") + _sha256(catalog_sha256, "catalog_sha256") + _sha256(reference_manifest_sha256, "reference_manifest_sha256") + _sha256(capability.get("sha256"), "capability_matrix.sha256") + capability_revision = _positive_int(capability.get("revision"), "capability_matrix.revision") + + rendered: list[dict[str, Any]] = [] + consumers: set[str] = set() + for deployment in deployments: + status = _text(deployment.get("status"), "deployment status") + if status == "BLOCKED": + rendered.append(_blocked_result( + deployment, + _text(deployment.get("blocked_reason"), "deployment blocked_reason"), + )) + continue + try: + history = _mapping(deployment.get("history"), "deployment history") + maxlen = _positive_int(history.get("maxlen"), "deployment history maxlen", maximum=10_000) + min_bars = _positive_int(history.get("min_bars"), "deployment history min_bars", maximum=10_000) + if min_bars > maxlen: + raise DeploymentBindingError("deployment history min_bars exceeds maxlen") + consumer_id = _consumer_id(deployment) + if consumer_id in consumers: + raise DeploymentBindingError("compiled consumer_id is duplicated") + consumer_class = _consumer_class(deployment) + routes = deployment.get("routes") + if not isinstance(routes, list) or not routes: + raise DeploymentBindingError("declared deployment routes are required") + products: list[dict[str, Any]] = [] + optional_unavailable: list[dict[str, Any]] = [] + for route_raw in routes: + route = _mapping(route_raw, "deployment route") + required = _bool(route.get("required", True), "route required") + identity = tuple( + _text(route.get(field), f"route {field}").upper() + for field in ("venue", "market", "product_type", "native_symbol") + ) + if identity[0] not in _V2_VENUES: + raise DeploymentBindingError("route venue is outside V2") + instrument = records.get(identity) + if instrument is None: + raise DeploymentBindingError( + "CATALOG_IDENTITY_UNAVAILABLE:" + ":".join(identity) + ) + missing = _missing_reason( + route=route, + instrument=instrument, + bindings=bindings, + reference_keys=reference_keys, + ) + product = _route_product( + deployment=deployment, + route=route, + instrument=instrument, + consumer_id=consumer_id, + consumer_class=consumer_class, + ) + if missing is not None: + if required: + raise DeploymentBindingError(missing) + optional_unavailable.append(_unavailable_optional(route, missing)) + continue + products.append(product) + if not products: + raise DeploymentBindingError("NO_ADMITTED_REQUIRED_ROUTE") + if len({ + (row["venue"], row["market"], row["product_type"], row["native_symbol"], row["feed"], row["interval"]) + for row in products + }) != len(products): + raise DeploymentBindingError("deployment routes are duplicated") + binding_without_digest = { + "schema": "qdl.v2.consumer-route-binding.v1", + "contract_version": CONTRACT_VERSION, + "consumer_id": consumer_id, + "consumer_class": consumer_class, + "release_revision": release_revision, + # Phase B is source-only. This anchors the binding to the + # exact approved release-routing generation; Phase C replaces + # it with the sealed runtime generation before mount. + "universal_manifest_sha256": release_routing_sha256, + "policy_sha256": policy_sha256, + "capability_matrix": { + "sha256": _sha256(capability.get("sha256"), "capability_matrix.sha256"), + "revision": capability_revision, + }, + "inventory_sha256": _inventory_digest(inventory), + "v1_rollback": {key: _text(rollback[key], f"v1_rollback.{key}") for key in sorted(rollback)}, + "independent_v1_venues": ["DNSE"], + "products": sorted(products, key=lambda row: row["requirement_id"]), + } + binding = { + **binding_without_digest, + "binding_sha256": _digest(binding_without_digest), + } + # Reuse the strict in-repository parser; TS/SDK use the same + # canonical field/digest contract independently. + ConsumerRouteBinding.from_canonical_mapping(binding) + consumers.add(consumer_id) + rendered.append({ + "deployment_id": _text(deployment.get("deployment_id"), "deployment_id"), + "alpha_id": _text(deployment.get("alpha_id"), "alpha_id"), + "status": "ADMITTED", + "reason": None, + "history": {"maxlen": maxlen, "min_bars": min_bars}, + "optional_unavailable": sorted( + optional_unavailable, + key=lambda item: ( + item["venue"], item["market"], item["product_type"], + item["native_symbol"], item["feed"], item["interval"] or "", + item["source_policy_id"], + ), + ), + "binding": binding, + }) + except DeploymentBindingError as error: + rendered.append(_blocked_result(deployment, str(error))) + + report_without_digest = { + "schema": REPORT_SCHEMA, + "contract_version": CONTRACT_VERSION, + "inventory_sha256": _inventory_digest(inventory), + "catalog_sha256": catalog_sha256, + "reference_manifest_sha256": reference_manifest_sha256, + "release_routing_sha256": release_routing_sha256, + "policy_sha256": policy_sha256, + "deployments": sorted(rendered, key=lambda item: item["deployment_id"]), + } + return {**report_without_digest, "compilation_sha256": _digest(report_without_digest)} + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> bool: + rendered = json.dumps(payload, indent=2, sort_keys=True).encode("utf-8") + b"\n" + if path.exists() and path.read_bytes() == rendered: + return False + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(rendered) + return True + + +def write_compilation(output_dir: Path, report: Mapping[str, Any]) -> tuple[str, ...]: + """Write only named derived artifacts to a caller-owned output directory.""" + + changed: list[str] = [] + for deployment in report["deployments"]: + binding = deployment.get("binding") + if binding is None: + continue + name = _SAFE_FILE_COMPONENT.sub("-", str(deployment["deployment_id"])).strip("-") + path = output_dir / f"{name}.binding.json" + if _write_json(path, binding): + changed.append(str(path)) + if _write_json(output_dir / "compilation-report.json", report): + changed.append(str(output_dir / "compilation-report.json")) + return tuple(changed) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--inventory", type=Path, required=True) + parser.add_argument("--catalog", type=Path, default=ROOT / "config/v2/stable-source-bindings.yaml") + parser.add_argument("--reference-manifest", type=Path, default=ROOT / "consumers/stable/reference-l2-stable.yaml") + parser.add_argument("--release-routing", type=Path, default=ROOT / "config/v2/stable-v2-release-routing.yaml") + parser.add_argument("--policy", type=Path, default=ROOT / "config/v2/universal-release-policy.yaml") + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--write", action="store_true") + args = parser.parse_args(argv) + if args.write and args.output_dir is None: + raise DeploymentBindingError("--write requires --output-dir") + + report = compile_inventory( + inventory=_load_json_mapping(args.inventory), + catalog=_load_mapping(args.catalog), + reference_manifest=_load_mapping(args.reference_manifest), + release_routing=_load_mapping(args.release_routing), + release_routing_sha256=_sha256_file(args.release_routing), + policy=_load_mapping(args.policy), + policy_sha256=_sha256_file(args.policy), + catalog_sha256=_sha256_file(args.catalog), + reference_manifest_sha256=_sha256_file(args.reference_manifest), + ) + changed = write_compilation(args.output_dir, report) if args.write else () + summary = { + "status": "WRITTEN_SOURCE_ONLY" if args.write else "DRY_RUN_SOURCE_ONLY", + "admitted": sum(item["status"] == "ADMITTED" for item in report["deployments"]), + "blocked": sum(item["status"] == "BLOCKED" for item in report["deployments"]), + "compilation_sha256": report["compilation_sha256"], + "changed_files": list(changed), + "runtime_mutations": 0, + "order_actions": 0, + } + print(json.dumps(summary, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/converge_v2_primary_runtime.py b/scripts/converge_v2_primary_runtime.py new file mode 100644 index 00000000..983035b1 --- /dev/null +++ b/scripts/converge_v2_primary_runtime.py @@ -0,0 +1,431 @@ +#!/usr/bin/env python3 +"""Converge a legacy partial V2 runtime bundle to the canonical compiler output. + +The initial Rust-primary packet intentionally mounted a compact realtime +subset. The current V2 catalog and acquisition plan contain the approved +multi-venue demand set, but a config-only replay of the old packet leaves +declared routes without their physical ingestion/core bindings. This tool is +the narrow, reusable repair: it regenerates five already-mounted JSON files +from the canonical compiler while preserving the active authority byte-for-byte. + +It is dry-run by default. ``--apply`` atomically replaces only three core +configs and two native-ingestor configs after writing exact rollback bytes to a +new private state directory. It never talks to a provider, Docker, Kafka, +Redis, SQLite, V1, Trading System, an alpha or an order path. +""" + +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import os +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, Mapping + + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_deployment import ( + StableAcquisitionPlan, + validate_shared_authority_record, + write_stable_runtime_bundle, +) + + +CONFIRM = "CONVERGE_QDL_V2_PRIMARY_RUNTIME" +DEFAULT_STATE_ROOT = Path("/home/bobby/.local/state/qdl-v2") +CORE_FILES = ("core.json", "core-002.json", "core-003.json") +INGESTOR_FILES = ("ingestor-binance-usdm.json", "ingestor-okx-swap.json") +RUNTIME_FILES = CORE_FILES + INGESTOR_FILES +_ALLOWED_CORE_DEDUP = frozenset({1_000_000, 100_000}) +_EXPECTED_CORE_DEDUP = 100_000 +_LINEAGE_FIELDS = frozenset({"instrument_catalog_revision", "instrument_revision"}) +_INGESTOR_ADDITIVE_FIELDS = frozenset({ + "config_revision", + "session_liveness_dir", + "session_liveness_write_interval_ms", +}) +_FIVE_LIQUID_PERPETUAL_BOOK_IDS = frozenset({ + "binance-usdm-btcusdt-book-primary-v2", + "binance-usdm-ethusdt-book-primary-v2", + "binance-usdm-solusdt-book-primary-v2", + "binance-usdm-dogeusdt-book-primary-v2", + "binance-usdm-bnbusdt-book-primary-v2", + "okx-swap-btc-usdt-swap-book-primary-v2", + "okx-swap-eth-usdt-swap-book-primary-v2", + "okx-swap-sol-usdt-swap-book-primary-v2", + "okx-swap-doge-usdt-swap-book-primary-v2", + "okx-swap-bnb-usdt-swap-book-primary-v2", +}) +_FIVE_LIQUID_BOOK_IDS_BY_INGESTOR = { + "ingestor-binance-usdm.json": frozenset({ + "binance-usdm-btcusdt-book-primary-v2", + "binance-usdm-ethusdt-book-primary-v2", + "binance-usdm-solusdt-book-primary-v2", + "binance-usdm-dogeusdt-book-primary-v2", + "binance-usdm-bnbusdt-book-primary-v2", + }), + "ingestor-okx-swap.json": frozenset({ + "okx-swap-btc-usdt-swap-book-primary-v2", + "okx-swap-eth-usdt-swap-book-primary-v2", + "okx-swap-sol-usdt-swap-book-primary-v2", + "okx-swap-doge-usdt-swap-book-primary-v2", + "okx-swap-bnb-usdt-swap-book-primary-v2", + }), +} + + +def _sha256(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _canonical_bytes(value: Mapping[str, Any]) -> bytes: + return (json.dumps(dict(value), indent=2, sort_keys=True) + "\n").encode("utf-8") + + +def _read_json(path: Path, *, field: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValueError(f"{field} is unreadable: {path}") from error + if not isinstance(value, dict): + raise ValueError(f"{field} must be a JSON object") + return value + + +def _without(value: Mapping[str, Any], *fields: str) -> dict[str, Any]: + result = copy.deepcopy(dict(value)) + for field in fields: + result.pop(field, None) + return result + + +def _binding_map( + bindings: object, + *, + key_field: str, + field: str, +) -> dict[str, dict[str, Any]]: + if not isinstance(bindings, list) or not bindings: + raise ValueError(f"{field} bindings are invalid") + result: dict[str, dict[str, Any]] = {} + for item in bindings: + if not isinstance(item, dict): + raise ValueError(f"{field} has a non-object binding") + key = item.get(key_field) + if not isinstance(key, str) or not key or key in result: + raise ValueError(f"{field} has an invalid/duplicate {key_field}") + result[key] = dict(item) + return result + + +def _lineage_equal(active: Mapping[str, Any], expected: Mapping[str, Any]) -> bool: + return _without(active, *_LINEAGE_FIELDS) == _without(expected, *_LINEAGE_FIELDS) + + +def _expected_runtime( + *, + authority: Mapping[str, Any], + catalog_path: Path, + acquisition_path: Path, +) -> dict[str, dict[str, Any]]: + catalog = StableSourceCatalog.load(catalog_path) + acquisition = StableAcquisitionPlan.load(acquisition_path, catalog=catalog) + with tempfile.TemporaryDirectory(prefix="qdl-primary-runtime-converge-") as raw: + generated = Path(raw) / "runtime" + write_stable_runtime_bundle( + generated, + catalog=catalog, + acquisition=acquisition, + authority=authority, + ) + return { + name: _read_json(generated / name, field=f"generated {name}") + for name in RUNTIME_FILES + } + + +def _validate_core( + *, + active: Mapping[str, Any], + expected: Mapping[str, Any], + file_name: str, +) -> dict[str, Any]: + active_outer = _without(active, "core") + expected_outer = _without(expected, "core") + if active_outer != expected_outer: + raise ValueError(f"{file_name} changes non-core runtime configuration") + active_core = active.get("core") + expected_core = expected.get("core") + if not isinstance(active_core, Mapping) or not isinstance(expected_core, Mapping): + raise ValueError(f"{file_name} lacks core configuration") + active_core_static = _without(active_core, "bindings", "dedup_capacity") + expected_core_static = _without(expected_core, "bindings", "dedup_capacity") + if active_core_static != expected_core_static: + raise ValueError(f"{file_name} changes non-binding core configuration") + active_dedup = active_core.get("dedup_capacity") + expected_dedup = expected_core.get("dedup_capacity") + if ( + not isinstance(active_dedup, int) + or active_dedup not in _ALLOWED_CORE_DEDUP + or expected_dedup != _EXPECTED_CORE_DEDUP + ): + raise ValueError(f"{file_name} has an unsupported dedup transition") + + active_bindings = _binding_map( + active_core.get("bindings"), key_field="source_id", field=f"active {file_name}" + ) + expected_bindings = _binding_map( + expected_core.get("bindings"), key_field="source_id", field=f"expected {file_name}" + ) + unknown = sorted(active_bindings.keys() - expected_bindings.keys()) + if unknown: + raise ValueError(f"{file_name} has bindings absent from canonical catalog: {unknown}") + drift = sorted( + source_id + for source_id, binding in active_bindings.items() + if not _lineage_equal(binding, expected_bindings[source_id]) + ) + if drift: + raise ValueError(f"{file_name} has retained binding semantic drift: {drift}") + added = sorted(expected_bindings.keys() - active_bindings.keys()) + missing_liquid_books = sorted( + _FIVE_LIQUID_PERPETUAL_BOOK_IDS - expected_bindings.keys() + ) + if missing_liquid_books: + raise ValueError(f"{file_name} lacks five-liquid perpetual L2 scope: {missing_liquid_books}") + return { + "before_binding_count": len(active_bindings), + "after_binding_count": len(expected_bindings), + "added_binding_count": len(added), + "added_five_liquid_book_source_ids": sorted( + _FIVE_LIQUID_PERPETUAL_BOOK_IDS & set(added) + ), + "retained_lineage_update_count": sum( + active_bindings[source_id] != expected_bindings[source_id] + for source_id in active_bindings + ), + "dedup_capacity": {"before": active_dedup, "after": expected_dedup}, + } + + +def _validate_ingestor( + *, + active: Mapping[str, Any], + expected: Mapping[str, Any], + file_name: str, +) -> dict[str, Any]: + active_static = _without(active, "bindings", *_INGESTOR_ADDITIVE_FIELDS) + expected_static = _without(expected, "bindings", *_INGESTOR_ADDITIVE_FIELDS) + if active_static != expected_static: + raise ValueError(f"{file_name} changes non-binding runtime configuration") + unexpected_removed = sorted( + set(active.keys()) - set(expected.keys()) - {"config_revision"} + ) + if unexpected_removed: + raise ValueError(f"{file_name} has unsupported active fields: {unexpected_removed}") + expected_liveness = { + field: expected.get(field) + for field in ("session_liveness_dir", "session_liveness_write_interval_ms") + } + if ( + not isinstance(expected_liveness["session_liveness_dir"], str) + or not expected_liveness["session_liveness_dir"] + or expected_liveness["session_liveness_write_interval_ms"] != 1_000 + ): + raise ValueError(f"{file_name} lacks bounded session-liveness configuration") + + active_bindings = _binding_map( + active.get("bindings"), key_field="subscription_id", field=f"active {file_name}" + ) + expected_bindings = _binding_map( + expected.get("bindings"), key_field="subscription_id", field=f"expected {file_name}" + ) + unknown = sorted(active_bindings.keys() - expected_bindings.keys()) + if unknown: + raise ValueError(f"{file_name} has bindings absent from canonical catalog: {unknown}") + drift = sorted( + subscription_id + for subscription_id, binding in active_bindings.items() + if not _lineage_equal(binding, expected_bindings[subscription_id]) + ) + if drift: + raise ValueError(f"{file_name} has retained binding semantic drift: {drift}") + added = sorted(expected_bindings.keys() - active_bindings.keys()) + book_ids = { + item["subscription_id"] + for item in expected_bindings.values() + if item.get("feed") == "BOOK" + } + required_books = _FIVE_LIQUID_BOOK_IDS_BY_INGESTOR[file_name] + if not required_books <= book_ids: + missing = sorted(required_books - book_ids) + raise ValueError(f"{file_name} lacks five-liquid native book scope: {missing}") + return { + "before_binding_count": len(active_bindings), + "after_binding_count": len(expected_bindings), + "added_binding_count": len(added), + "added_book_subscription_ids": sorted(book_ids & set(added)), + "retained_lineage_update_count": sum( + active_bindings[subscription_id] != expected_bindings[subscription_id] + for subscription_id in active_bindings + ), + "config_revision": { + "before": active.get("config_revision"), + "after": expected.get("config_revision"), + }, + "session_liveness_write_interval_ms": expected_liveness[ + "session_liveness_write_interval_ms" + ], + } + + +def _atomic_replace(path: Path, content: bytes, *, mode: int) -> None: + temporary = path.with_name(f".{path.name}.qdl-primary-converge-{os.getpid()}-{time.time_ns()}") + try: + with temporary.open("xb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.chmod(temporary, mode) + os.replace(temporary, path) + directory_fd = os.open(path.parent, os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + finally: + temporary.unlink(missing_ok=True) + + +def converge( + *, + runtime_dir: Path, + output_dir: Path | None, + apply: bool, + catalog_path: Path = ROOT / "config/v2/stable-source-bindings.yaml", + acquisition_path: Path = ROOT / "config/v2/stable-acquisition-bindings.yaml", + state_root: Path = DEFAULT_STATE_ROOT, +) -> dict[str, Any]: + runtime_dir = runtime_dir.resolve() + state_root = state_root.resolve() + if not runtime_dir.is_dir(): + raise ValueError("runtime directory is invalid") + if apply: + if output_dir is None: + raise ValueError("apply requires an output directory") + output_dir = output_dir.resolve() + if output_dir.exists() or state_root not in (output_dir, *output_dir.parents): + raise ValueError("output directory must be a new private QDL state path") + + authority_path = runtime_dir / "authority.json" + authority_bytes = authority_path.read_bytes() + authority = _read_json(authority_path, field="active authority") + try: + validate_shared_authority_record(authority) + except ValueError as error: + raise ValueError("active authority is invalid") from error + expected = _expected_runtime( + authority=authority, + catalog_path=catalog_path, + acquisition_path=acquisition_path, + ) + + pending: dict[str, tuple[Path, bytes, bytes, int, dict[str, Any]]] = {} + for file_name in CORE_FILES: + path = runtime_dir / file_name + active_bytes = path.read_bytes() + active = _read_json(path, field=f"active {file_name}") + change = _validate_core(active=active, expected=expected[file_name], file_name=file_name) + pending[file_name] = ( + path, active_bytes, _canonical_bytes(expected[file_name]), + path.stat().st_mode & 0o777, change, + ) + for file_name in INGESTOR_FILES: + path = runtime_dir / file_name + active_bytes = path.read_bytes() + active = _read_json(path, field=f"active {file_name}") + change = _validate_ingestor(active=active, expected=expected[file_name], file_name=file_name) + pending[file_name] = ( + path, active_bytes, _canonical_bytes(expected[file_name]), + path.stat().st_mode & 0o777, change, + ) + + files = { + name: { + "before_sha256": _sha256(active), + "after_sha256": _sha256(rendered), + "before_mode": oct(mode), + "changed": active != rendered, + **change, + } + for name, (_path, active, rendered, mode, change) in pending.items() + } + if not all(item["changed"] for item in files.values()): + raise ValueError("all five runtime files must require canonical convergence") + result: dict[str, Any] = { + "schema": "qdl.v2.primary-runtime-convergence.v1", + "status": "APPLIED" if apply else "DRY_RUN", + "runtime_dir": str(runtime_dir), + "authority_sha256": _sha256(authority_bytes), + "authority_bytes_preserved": True, + "files": files, + "production_mutations": 0 if not apply else len(RUNTIME_FILES), + } + if not apply: + return result + + assert output_dir is not None + rollback_dir = output_dir / "rollback" + output_dir.mkdir(parents=True, mode=0o700) + rollback_dir.mkdir(mode=0o700) + for file_name, (_path, active, _rendered, mode, _change) in pending.items(): + backup = rollback_dir / file_name + backup.write_bytes(active) + backup.chmod(mode) + + applied: list[str] = [] + try: + for file_name, (path, _active, rendered, mode, _change) in pending.items(): + _atomic_replace(path, rendered, mode=mode) + applied.append(file_name) + except Exception: + for file_name in reversed(applied): + path, active, _rendered, mode, _change = pending[file_name] + _atomic_replace(path, active, mode=mode) + raise + + result["rollback_dir"] = str(rollback_dir) + receipt = output_dir / "receipt.json" + receipt.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + receipt.chmod(0o640) + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--runtime-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--apply", action="store_true") + parser.add_argument("--confirm") + args = parser.parse_args(argv) + if args.apply and args.confirm != CONFIRM: + raise SystemExit(f"--apply requires --confirm {CONFIRM}") + print(json.dumps(converge( + runtime_dir=args.runtime_dir, + output_dir=args.output_dir, + apply=args.apply, + ), indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/materialize_alpha_v2_identity.py b/scripts/materialize_alpha_v2_identity.py new file mode 100644 index 00000000..024893a0 --- /dev/null +++ b/scripts/materialize_alpha_v2_identity.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Atomically materialize protected V2 identity files for a non-root alpha. + +The source identity remains owned by the operator and is never mounted into an +alpha container. This helper copies only the required client material to one +workload-scoped mount whose permissions match the runtime UID/GID. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import uuid +from typing import Mapping + + +CONFIRM = "MATERIALIZE_QDL_ALPHA_V2_IDENTITY" +DEFAULT_STATE_ROOT = Path("/home/bobby/.local/state/qdl-v2") +_WORKLOAD_ID = re.compile(r"[a-z0-9][a-z0-9._-]{2,120}\Z") +_TLS_FILES = { + "ca.crt": 0o444, + "client.crt": 0o444, + "client.key": 0o400, +} +_JWT_FILES = { + "private.key": 0o400, + "public.pem": 0o444, +} + + +class IdentityMaterializationError(RuntimeError): + """Raised when a protected identity cannot be safely materialized.""" + + +def _sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _regular_file_names(path: Path, *, field: str) -> set[str]: + if not path.is_dir() or path.is_symlink(): + raise IdentityMaterializationError(f"{field} must be a real directory") + values: set[str] = set() + for child in path.iterdir(): + if child.is_symlink() or not child.is_file(): + raise IdentityMaterializationError(f"{field} contains a non-regular file") + values.add(child.name) + return values + + +def _validate_source(path: Path, *, expected: Mapping[str, int], field: str) -> None: + actual = _regular_file_names(path, field=field) + if actual != set(expected): + raise IdentityMaterializationError( + f"{field} files differ: expected {sorted(expected)}, got {sorted(actual)}" + ) + for name in expected: + source = path / name + if source.stat().st_size <= 0: + raise IdentityMaterializationError(f"{field}/{name} is empty") + + +def _validate_target( + *, state_root: Path, workload_id: str +) -> tuple[Path, Path]: + if _WORKLOAD_ID.fullmatch(workload_id) is None: + raise IdentityMaterializationError("workload id is invalid") + root = state_root.resolve() + if not root.is_dir() or root.is_symlink(): + raise IdentityMaterializationError("state root must be a real existing directory") + identities_root = root / "workload-identities" + target = identities_root / workload_id + if target.exists() or target.is_symlink(): + raise FileExistsError(f"identity target already exists: {target}") + return identities_root, target + + +def _copy_mode( + *, source: Path, destination: Path, mode: int, uid: int, gid: int +) -> None: + shutil.copyfile(source, destination) + os.chown(destination, uid, gid) + destination.chmod(mode) + + +def _materialize_directory( + *, + source: Path, + destination: Path, + files: Mapping[str, int], + uid: int, + gid: int, +) -> None: + destination.mkdir(mode=0o500) + os.chown(destination, uid, gid) + destination.chmod(0o500) + for name, mode in files.items(): + _copy_mode( + source=source / name, + destination=destination / name, + mode=mode, + uid=uid, + gid=gid, + ) + + +def _manifest( + *, workload_id: str, tls_source: Path, jwt_source: Path, uid: int, gid: int +) -> dict[str, object]: + return { + "schema": "qdl.v2.alpha-workload-identity.v1", + "workload_id": workload_id, + "workload_uid": uid, + "workload_gid": gid, + "tls": { + name: {"sha256": _sha256_file(tls_source / name), "mode": format(mode, "04o")} + for name, mode in sorted(_TLS_FILES.items()) + }, + "jwt": { + name: {"sha256": _sha256_file(jwt_source / name), "mode": format(mode, "04o")} + for name, mode in sorted(_JWT_FILES.items()) + }, + "secret_values_recorded": False, + } + + +def materialize_identity( + *, + tls_source: Path, + jwt_source: Path, + state_root: Path = DEFAULT_STATE_ROOT, + workload_id: str, + workload_uid: int, + workload_gid: int, + apply: bool, +) -> dict[str, object]: + """Validate and optionally atomically create one workload-scoped mount.""" + + if workload_uid < 1 or workload_gid < 1: + raise IdentityMaterializationError("workload uid and gid must be positive") + _validate_source(tls_source, expected=_TLS_FILES, field="tls source") + _validate_source(jwt_source, expected=_JWT_FILES, field="jwt source") + identities_root, target = _validate_target( + state_root=state_root, workload_id=workload_id + ) + manifest = _manifest( + workload_id=workload_id, + tls_source=tls_source, + jwt_source=jwt_source, + uid=workload_uid, + gid=workload_gid, + ) + result = { + "status": "APPLIED" if apply else "DRY_RUN", + "target": str(target), + "tls_mount": str(target / "tls"), + "jwt_mount": str(target / "jwt"), + "manifest_sha256": hashlib.sha256( + json.dumps(manifest, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest(), + "secret_values_recorded": False, + "runtime_mutations": 0, + "order_actions": 0, + } + if not apply: + return result + if os.geteuid() != 0: + raise PermissionError("apply requires effective root to set workload ownership") + + root_stat = state_root.resolve().stat() + identities_root.mkdir(mode=0o750, exist_ok=True) + os.chown(identities_root, root_stat.st_uid, root_stat.st_gid) + identities_root.chmod(0o750) + staging = identities_root / f".{workload_id}.staging-{uuid.uuid4().hex}" + try: + staging.mkdir(mode=0o750) + os.chown(staging, root_stat.st_uid, root_stat.st_gid) + staging.chmod(0o750) + _materialize_directory( + source=tls_source, + destination=staging / "tls", + files=_TLS_FILES, + uid=workload_uid, + gid=workload_gid, + ) + _materialize_directory( + source=jwt_source, + destination=staging / "jwt", + files=_JWT_FILES, + uid=workload_uid, + gid=workload_gid, + ) + manifest_path = staging / "identity-manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + os.chown(manifest_path, root_stat.st_uid, root_stat.st_gid) + manifest_path.chmod(0o640) + staging.rename(target) + except Exception: + shutil.rmtree(staging, ignore_errors=True) + raise + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--tls-source", type=Path, required=True) + parser.add_argument("--jwt-source", type=Path, required=True) + parser.add_argument("--state-root", type=Path, default=DEFAULT_STATE_ROOT) + parser.add_argument("--workload-id", required=True) + parser.add_argument("--workload-uid", type=int, default=1000) + parser.add_argument("--workload-gid", type=int, default=1000) + parser.add_argument("--apply", action="store_true") + parser.add_argument("--confirm") + args = parser.parse_args(argv) + if args.apply and args.confirm != CONFIRM: + raise RuntimeError(f"--apply requires --confirm {CONFIRM}") + result = materialize_identity( + tls_source=args.tls_source, + jwt_source=args.jwt_source, + state_root=args.state_root, + workload_id=args.workload_id, + workload_uid=args.workload_uid, + workload_gid=args.workload_gid, + apply=args.apply, + ) + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/scripts/phase103_consumer_receipt_acceptance.py b/scripts/phase103_consumer_receipt_acceptance.py index 58bdc1d3..b47ba43f 100644 --- a/scripts/phase103_consumer_receipt_acceptance.py +++ b/scripts/phase103_consumer_receipt_acceptance.py @@ -20,15 +20,10 @@ from contextlib import asynccontextmanager, contextmanager from dataclasses import dataclass, replace from pathlib import Path -from typing import AsyncIterator, Iterator +from typing import AsyncIterator, Callable, Iterator ROOT = Path(__file__).resolve().parents[1] -# A historical alpha BAR seed normally replays one retained event. A bounded -# repair can append a few older canonical records after that seed, so C2 must -# consume through the strict snapshot watermark rather than assume one event -# is enough. This is acceptance-only, never an unbounded consumer catch-up. -_MAX_HISTORICAL_REPLAY_CATCHUP_EVENTS = 16 if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) @@ -45,7 +40,8 @@ validate_resume_offsets, warmup_content_fingerprint, ) -from qdl.adapters.intervals import canonical_interval_ms +from qdl.adapters.intervals import canonical_interval_ms, is_valid_bar_open_ms +from qdl.runtime.stable_bar_edge import durable_bar_history_capacity_rows from qdl.runtime.stable_catalog import StableSourceCatalog from qdl.runtime.stable_deployment import StableAcquisitionPlan from qdl.certification.phase105_release_observations import compact_view_quality @@ -74,7 +70,18 @@ DEFAULT_TRADING_MANIFEST = ROOT / "consumers/stable/trading-system-paper.yaml" DEFAULT_ALPHA_MANIFEST = ROOT / "consumers/stable/alpha-binance-paper.yaml" _QUIET_QUOTE_RETRY_SECONDS = 0.2 -_QUIET_TRADE_STREAM_OBSERVATION_SECONDS = 2.0 +_BOOK_SNAPSHOT_RETRY_SECONDS = 1.0 +_TRANSIENT_SESSION_RETRY_SECONDS = 0.5 +_QUIET_CONTINUITY_STREAM_OBSERVATION_SECONDS = 2.0 +_QUIET_CONTINUITY_FEEDS = frozenset({"TRADE", "BOOK_DELTA"}) +_TRANSIENT_PROVIDER_SESSION_STATES = frozenset({"UNKNOWN", "STALE", "DISCONNECTED"}) +_TRANSIENT_SESSION_QUALITY_STATES = frozenset( + {"STARTING", "CONNECTING", "SUBSCRIBING", "SYNCING", "RESYNCING", "STALE", "DEGRADED"} +) +# C2 proves a representative retained BAR window, not an impossible request +# for the full per-consumer quota on every calendar interval. Production +# callers still declare their own bounded maxlen through the public SDK. +_C2_BAR_WARMUP_ROWS = 700 @dataclass(frozen=True, slots=True) @@ -86,6 +93,65 @@ class WorkloadIdentity: credential: RotatingJwtCredentialProvider +def compact_feed_status(status: object) -> dict[str, object]: + """Return bounded typed diagnostic evidence without a market payload.""" + + quality = getattr(status, "quality", None) + instrument_uid = getattr(status, "instrument_uid", None) + feed = getattr(status, "feed", None) + if not isinstance(instrument_uid, str) or not instrument_uid: + raise ValueError("C2 feed-status evidence lacks an instrument identity") + feed_value = getattr(feed, "value", feed) + if not isinstance(feed_value, str) or not feed_value: + raise ValueError("C2 feed-status evidence lacks a feed") + fields = { + "state": getattr(quality, "state", None), + "event_recency_state": getattr(quality, "event_recency_state", None), + "event_age_ms": getattr(quality, "freshness_ms", None), + "provider_session_state": getattr(quality, "provider_session_state", None), + "provider_session_liveness_ms": getattr(quality, "provider_session_liveness_ms", None), + "gap_open": getattr(quality, "gap_open", None), + "complete": getattr(quality, "complete", None), + "execution_eligible": getattr(quality, "execution_eligible", None), + "policy_id": getattr(quality, "policy_id", None), + } + liveness = fields["provider_session_liveness_ms"] + if ( + not isinstance(fields["state"], str) + or not isinstance(fields["event_recency_state"], str) + or not isinstance(fields["event_age_ms"], int) + or fields["event_age_ms"] < 0 + or not isinstance(fields["provider_session_state"], str) + or liveness is not None and (not isinstance(liveness, int) or liveness < 0) + or not isinstance(fields["gap_open"], bool) + or not isinstance(fields["complete"], bool) + or not isinstance(fields["execution_eligible"], bool) + or not isinstance(fields["policy_id"], str) + or not fields["policy_id"] + ): + raise ValueError("C2 feed-status evidence has invalid quality fields") + flags = getattr(quality, "flags", None) + if not isinstance(flags, list) or any(not isinstance(item, str) for item in flags): + raise ValueError("C2 feed-status evidence has invalid flags") + return { + "schema": "qdl.c2.feed-status-evidence.v1", + "instrument_uid": instrument_uid, + "feed": feed_value, + "quality": fields, + "flags": sorted(set(flags))[:16], + "payload_recorded": False, + } + + +class C2StatusEvidenceError(ContinuityError): + """Strict-read failure with payload-free typed status evidence.""" + + def __init__(self, code: str, detail: str, *, status: object) -> None: + super().__init__(code, detail) + self.status_evidence = compact_feed_status(status) + self.replica: str | None = None + + def _identity( *, product: AcceptanceProduct, @@ -150,6 +216,32 @@ def _client( ) +def _receipt_client( + identity: WorkloadIdentity, + *, + base_url: str, + grpc_target: str, + cursor_path: Path, + timeout_seconds: float, + client_factory: Callable[..., AsyncDataLayerClient] | None, +) -> AsyncDataLayerClient: + """Create a receipt client, optionally through an acceptance-only wrapper. + + Production SDK construction remains `_client`. C2 may supply a local + wrapper for its own bounded request budget; the wrapper never changes the + public client contract, identity, cursor or stream transport. + """ + + factory = _client if client_factory is None else client_factory + return factory( + identity, + base_url=base_url, + grpc_target=grpc_target, + cursor_path=cursor_path, + timeout_seconds=timeout_seconds, + ) + + def _cursor_path(state_dir: Path, product: AcceptanceProduct) -> Path: identity = "|".join(product.identity).encode() return state_dir / f"{hashlib.sha256(identity).hexdigest()}.json" @@ -194,7 +286,40 @@ def _uses_historical_bar_replay(product: AcceptanceProduct) -> bool: ) -def _historical_bar_replay_requirement(requirement, *, latest_open_time_ns: int): +def _c2_requirement(requirement): + """Keep the C2 BAR history proof bounded without reducing public quota. + + The registered manifest quota remains the caller's `1..10,000` ceiling. + A fixed C2 proof must instead fit the real retained final-BAR window across + all native intervals; otherwise a 12-hour or weekly route would be asked + for decades of history merely because its client is allowed to request it. + """ + + if requirement.feed.value != "BAR": + return requirement + specification = requirement.warmup_specification + if specification is None or specification.rows is None: + raise ValueError("C2 BAR product requires a row-bounded warmup policy") + if not requirement.interval: + raise ValueError("C2 BAR product requires an interval") + rows = min( + _C2_BAR_WARMUP_ROWS, + specification.rows, + durable_bar_history_capacity_rows(requirement.interval), + ) + return replace( + requirement, + warmup_limit=rows, + warmup=(SdkWarmupSpecification(rows=rows) if requirement.warmup is not None else None), + ) + + +def _historical_bar_replay_requirement( + requirement, + *, + latest_open_time_ns: int, + calendar_provider: str | None = None, +): """Build one aligned, bounded cursor seed before two retained BAR records. The caller has already checked the current strict requirement. This @@ -208,9 +333,15 @@ def _historical_bar_replay_requirement(requirement, *, latest_open_time_ns: int) if requirement.consumer_grade.value == "EXECUTION": raise ValueError("execution BAR receipts must use the strict live path") interval_ns = canonical_interval_ms(requirement.interval) * 1_000_000 + latest_open_ms, sub_millisecond_ns = divmod(latest_open_time_ns, 1_000_000) if ( latest_open_time_ns <= 2 * interval_ns - or latest_open_time_ns % interval_ns + or sub_millisecond_ns + or not is_valid_bar_open_ms( + requirement.interval, + latest_open_ms, + provider=calendar_provider, + ) ): raise ValueError("latest BAR open time cannot form a retained replay seed") original_warmup = requirement.warmup_specification @@ -235,12 +366,56 @@ def _historical_bar_replay_requirement(requirement, *, latest_open_time_ns: int) ) -async def _next_data(session, *, timeout_seconds: float) -> tuple[StreamEvent, list[str]]: +async def _next_data( + session, + *, + timeout_seconds: float, + stream_open_timeout_seconds: float | None = None, +) -> tuple[StreamEvent, list[str]]: + """Read one data frame after a bounded stream-open and event window. + + C2's quota pacer may wait before a lazy SDK stream opens. That local wait + is not market-data latency, so it uses the separately bounded opening + window. Once the server begins its control handshake, the original data + event timeout resumes unchanged. + """ + + if stream_open_timeout_seconds is None: + controls: list[str] = [] + for _ in range(8): + item = await asyncio.wait_for(session.__anext__(), timeout=timeout_seconds) + if isinstance(item, ControlEvent): + controls.append(item.code) + continue + if isinstance(item, StreamEvent): + return item, controls + raise AssertionError("V2 SDK stream emitted an unknown event type") + raise AssertionError("V2 SDK stream emitted controls without market data") + if stream_open_timeout_seconds <= 0: + raise ValueError("stream-open timeout must be positive when declared") controls: list[str] = [] + open_deadline = time.monotonic() + max( + timeout_seconds, + stream_open_timeout_seconds or timeout_seconds, + ) + event_deadline: float | None = None for _ in range(8): - item = await asyncio.wait_for(session.__anext__(), timeout=timeout_seconds) + deadline = event_deadline if event_deadline is not None else open_deadline + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError + item = await asyncio.wait_for(session.__anext__(), timeout=remaining) if isinstance(item, ControlEvent): controls.append(item.code) + if item.code in {"RECONNECTED", "SNAPSHOT_REPLACED"}: + # The SDK will lazily open a fresh stream on its next read. + event_deadline = None + open_deadline = time.monotonic() + max( + timeout_seconds, + stream_open_timeout_seconds or timeout_seconds, + ) + else: + event_deadline = time.monotonic() + timeout_seconds continue if isinstance(item, StreamEvent): return item, controls @@ -252,11 +427,44 @@ async def _next_data_or_timeout( session, *, timeout_seconds: float, + stream_open_timeout_seconds: float | None = None, ) -> tuple[StreamEvent | None, list[str]]: - """Observe one stream without manufacturing data when a trade channel is quiet.""" + """Observe one stream without manufacturing data when a continuity feed is quiet. + + The optional opening bound covers only a local C2 quota reservation before + the server emits its first handshake control. The original quiet-data + timeout begins immediately after that control; a silent or incomplete + server handshake therefore remains fail-closed. + """ + + if stream_open_timeout_seconds is None: + controls: list[str] = [] + deadline = time.monotonic() + timeout_seconds + for _ in range(8): + remaining = deadline - time.monotonic() + if remaining <= 0: + return None, controls + try: + item = await asyncio.wait_for(session.__anext__(), timeout=remaining) + except TimeoutError: + return None, controls + if isinstance(item, ControlEvent): + controls.append(item.code) + continue + if isinstance(item, StreamEvent): + return item, controls + raise AssertionError("V2 SDK stream emitted an unknown event type") + return None, controls + if stream_open_timeout_seconds <= 0: + raise ValueError("stream-open timeout must be positive when declared") controls: list[str] = [] - deadline = time.monotonic() + timeout_seconds + open_deadline = time.monotonic() + max( + timeout_seconds, + stream_open_timeout_seconds or timeout_seconds, + ) + event_deadline: float | None = None for _ in range(8): + deadline = event_deadline if event_deadline is not None else open_deadline remaining = deadline - time.monotonic() if remaining <= 0: return None, controls @@ -266,6 +474,16 @@ async def _next_data_or_timeout( return None, controls if isinstance(item, ControlEvent): controls.append(item.code) + if item.code in {"RECONNECTED", "SNAPSHOT_REPLACED"}: + # This is an SDK-local control. Its next read opens a new + # paced stream, so restart only the bounded open window. + event_deadline = None + open_deadline = time.monotonic() + max( + timeout_seconds, + stream_open_timeout_seconds or timeout_seconds, + ) + else: + event_deadline = time.monotonic() + timeout_seconds continue if isinstance(item, StreamEvent): return item, controls @@ -273,21 +491,43 @@ async def _next_data_or_timeout( return None, controls -def _allows_quiet_trade_observation(product: AcceptanceProduct, requirement) -> bool: - """Whether this no-order C2 probe may observe a quiet live trade channel.""" +def _allows_quiet_continuity_observation(product: AcceptanceProduct, requirement) -> bool: + """Whether C2 may observe a quiet, live continuity channel. + + BOOK_DELTA is not price data. It can be quiet while its verified book and + transport session stay healthy, so C2 records it as non-executable + continuity evidence rather than manufacturing a new update. + """ return ( product.delivery is DeliveryClass.DURABLE - and product.feed.value == "TRADE" + and product.feed.value in _QUIET_CONTINUITY_FEEDS and requirement.effective_event_recency_policy is SdkStalePolicy.OBSERVE and requirement.max_session_liveness_ms is not None ) -def _quiet_trade_status_is_observable(product: AcceptanceProduct, requirement, status) -> bool: - """Keep a quiet session distinct from fresh/executable market data.""" +def _allows_quiet_final_bar_handoff(product: AcceptanceProduct) -> bool: + """Allow a bounded no-event observation for retained non-execution BARs. + + A current alpha BAR channel can be quiet until the next interval closes. + C2 still proves signed stream controls and a fresh final BAR from each + replica; it never turns that quiet period into an execution price or a + durable replay checkpoint. Execution BARs keep the ordinary live-event + path. + """ + + return ( + product.delivery is DeliveryClass.DURABLE + and product.feed.value == "BAR" + and product.requirement.consumer_grade.value != "EXECUTION" + ) + + +def _quiet_continuity_status_is_observable(product: AcceptanceProduct, requirement, status) -> bool: + """Keep a quiet connected continuity channel distinct from price data.""" quality = status.quality return ( - _allows_quiet_trade_observation(product, requirement) + _allows_quiet_continuity_observation(product, requirement) and status.instrument_uid == requirement.instrument_uid and status.feed is requirement.feed and quality.policy_id == requirement.source_policy_id @@ -302,11 +542,11 @@ def _quiet_trade_status_is_observable(product: AcceptanceProduct, requirement, s ) -def _fresh_trade_status_is_observable(product: AcceptanceProduct, requirement, status) -> bool: - """Accept a live trade session that simply has no new print in this probe.""" +def _fresh_continuity_status_is_observable(product: AcceptanceProduct, requirement, status) -> bool: + """Accept a fresh verified continuity event during a bounded C2 probe.""" quality = status.quality return ( - _allows_quiet_trade_observation(product, requirement) + _allows_quiet_continuity_observation(product, requirement) and status.instrument_uid == requirement.instrument_uid and status.feed is requirement.feed and quality.policy_id == requirement.source_policy_id @@ -327,11 +567,11 @@ def _require_signed_cursor_controls(controls: list[str]) -> None: if missing: raise ContinuityError( "CURSOR_INVALID", - "C2 no-event TRADE observation did not confirm the signed cursor stream", + "C2 no-event continuity observation did not confirm the signed cursor stream", ) -async def _classify_no_event_trade_session( +async def _classify_no_event_continuity_session( client: AsyncDataLayerClient, *, product: AcceptanceProduct, @@ -345,18 +585,39 @@ async def _classify_no_event_trade_session( ) except TimeoutError as timeout: raise ContinuityError( - "DATA_STALE", "C2 quiet TRADE status did not return before its deadline" + "DATA_STALE", "C2 quiet continuity status did not return before its deadline" ) from timeout - if _fresh_trade_status_is_observable(product, requirement, status): + if _fresh_continuity_status_is_observable(product, requirement, status): return "FRESH_EXECUTABLE" - if _quiet_trade_status_is_observable(product, requirement, status): + if _quiet_continuity_status_is_observable(product, requirement, status): return "QUIET_NON_EXECUTABLE" raise ContinuityError( "DATA_STALE", - "C2 no-event TRADE observation requires a live fresh/executable or quiet/non-executable session", + "C2 no-event continuity observation requires a live fresh/executable or quiet/non-executable session", ) +async def _verify_quiet_final_bar_current( + client: AsyncDataLayerClient, + *, + product: AcceptanceProduct, + requirement, + timeout_seconds: float, +) -> str: + """Prove a quiet retained BAR remains final and current on this replica.""" + + if not _allows_quiet_final_bar_handoff(product): + raise ValueError("quiet final BAR verification requires a non-execution durable BAR") + current = await _strict_snapshot_for_c2( + client, + product=product, + requirement=requirement, + timeout_seconds=timeout_seconds, + ) + validate_product_view(product, current.data) + return "CURRENT_FINAL_BAR" + + def _stream_handoff_mode( product: AcceptanceProduct, *, @@ -369,19 +630,28 @@ def _stream_handoff_mode( if product.delivery is not DeliveryClass.DURABLE: return "NOT_APPLICABLE" if acknowledged_offset is None: + if len(no_event_sessions) == 2 and all( + item == "CURRENT_FINAL_BAR" for item in no_event_sessions + ): + return "CURRENT_FINAL_BAR_OBSERVED_NO_CURSOR" if ( len(no_event_sessions) == 2 and no_event_sessions[0] == "CURSOR_ACKNOWLEDGED" and no_event_sessions[1] in { "FRESH_EXECUTABLE_AFTER_CURSOR", "QUIET_NON_EXECUTABLE_AFTER_CURSOR", + "CURRENT_FINAL_BAR_AFTER_CURSOR", } ): return "SIGNED_CURSOR_REOPENED_NO_NEW_EVENT" if len(no_event_sessions) != 2: raise ValueError("C2 no-event stream evidence requires both sessions") if no_event_sessions[-1] == "EVENT_AFTER_REOPEN": - if no_event_sessions[0] not in {"FRESH_EXECUTABLE", "QUIET_NON_EXECUTABLE"}: + if no_event_sessions[0] not in { + "FRESH_EXECUTABLE", + "QUIET_NON_EXECUTABLE", + "CURRENT_FINAL_BAR", + }: raise ValueError("C2 no-event stream evidence has an invalid initial session") return "LIVE_EVENT_AFTER_REOPEN_NO_CURSOR" if all(item == "FRESH_EXECUTABLE" for item in no_event_sessions): @@ -394,6 +664,21 @@ def _stream_handoff_mode( return "DURABLE_CURSOR_REPLAYED" +def _replay_precedes_handoff(*, logical_offset: int, watermark_offset: int) -> bool: + """Whether a resumed frame restores state rather than supplies a live price. + + A reconnect checks a fresh query handoff, yet a durable cursor may correctly + point before that handoff. Those records are required for deterministic + state recovery. They retain identity, source, offset and gap validation, + but cannot be claimed as an executable current price; a strict read-back + is mandatory after the replay acknowledgement. + """ + + if logical_offset < 0 or watermark_offset < 0: + raise ValueError("durable replay offsets must be non-negative") + return logical_offset <= watermark_offset + + async def _query_product( product: AcceptanceProduct, *, @@ -427,8 +712,13 @@ async def _query_product_with_quality( dict[str, object] | None, ]: """Query both replicas and retain only compact quality evidence for B3.""" - requirement = sdk_requirement(product) + requirement = _c2_requirement(sdk_requirement(product)) bar_alignment: dict[str, object] | None = None + snapshot_timeout_seconds = ( + _stream_event_timeout_seconds(product, timeout_seconds) + if product.feed.value == "BOOK_SNAPSHOT" + else timeout_seconds + ) primary_started = time.perf_counter() if product.feed.value == "BAR": primary_response = await primary.warmup(requirement) @@ -439,12 +729,16 @@ async def _query_product_with_quality( validate_product_view(product, primary_view) primary_hash = warmup_content_fingerprint(primary_response.data) else: - primary_response = await _strict_snapshot_for_c2( - primary, - product=product, - requirement=requirement, - timeout_seconds=timeout_seconds, - ) + try: + primary_response = await _strict_snapshot_for_c2( + primary, + product=product, + requirement=requirement, + timeout_seconds=snapshot_timeout_seconds, + ) + except C2StatusEvidenceError as error: + error.replica = "primary" + raise primary_latency_ms = (time.perf_counter() - primary_started) * 1000 primary_view = primary_response.data validate_product_view(product, primary_view) @@ -469,12 +763,16 @@ async def _query_product_with_quality( else: validate_replica_views(product, primary_view, secondary_view) else: - secondary_response = await _strict_snapshot_for_c2( - secondary, - product=product, - requirement=requirement, - timeout_seconds=timeout_seconds, - ) + try: + secondary_response = await _strict_snapshot_for_c2( + secondary, + product=product, + requirement=requirement, + timeout_seconds=snapshot_timeout_seconds, + ) + except C2StatusEvidenceError as error: + error.replica = "secondary" + raise secondary_latency_ms = (time.perf_counter() - secondary_started) * 1000 secondary_view = secondary_response.data primary_hash, secondary_hash = validate_replica_views( @@ -545,6 +843,104 @@ def _fresh_quote_status_is_retryable( ) +def _fresh_trade_without_session_sla_is_retryable( + product: AcceptanceProduct, + requirement, + status, +) -> bool: + """Retry only a proven fresh TRADE race when no session SLA was declared. + + A manifest without max_session_liveness_ms intentionally reports + NOT_APPLICABLE rather than inventing transport liveness. This path does + not admit a stale trade: it merely lets the existing strict snapshot retry + after a typed status proves the same identity became current and executable. + """ + + quality = status.quality + return ( + product.feed.value == "TRADE" + and requirement.max_session_liveness_ms is None + and status.instrument_uid == requirement.instrument_uid + and status.feed is requirement.feed + and quality.policy_id == requirement.source_policy_id + and quality.state == "LIVE" + and quality.event_recency_state == "LIVE" + and quality.provider_session_state == "NOT_APPLICABLE" + and quality.provider_session_liveness_ms is None + and quality.complete + and not quality.gap_open + and quality.execution_eligible + ) + + +def _transitional_session_status_is_retryable( + product: AcceptanceProduct, + requirement, + status, +) -> bool: + """Allow one bounded C2 re-poll while an identity-matched session reconnects. + + This does not admit a stale price or book delta. It only keeps the strict + snapshot loop alive until its existing deadline, after which the SDK must + still independently return a fresh, live, complete and gap-free view. + """ + + quality = status.quality + return ( + product.feed.value in {"QUOTE", "BOOK_DELTA"} + and requirement.max_session_liveness_ms is not None + and status.instrument_uid == requirement.instrument_uid + and status.feed is requirement.feed + and quality.policy_id == requirement.source_policy_id + and quality.state in _TRANSIENT_SESSION_QUALITY_STATES + and quality.provider_session_state in _TRANSIENT_PROVIDER_SESSION_STATES + and quality.complete + and not quality.gap_open + and not quality.execution_eligible + ) + + +def _book_snapshot_status_is_refreshable( + product: AcceptanceProduct, + requirement, + status, +) -> bool: + """Permit a bounded retry while a verified book snapshot renews. + + The stale response remains rejected; only a subsequent strict read can pass. + Native session-backed and sessionless snapshot providers both retain their + declared liveness policy, identity, completeness and gap checks. + """ + + quality = status.quality + return ( + product.feed.value == "BOOK_SNAPSHOT" + and status.instrument_uid == requirement.instrument_uid + and status.feed is requirement.feed + and quality.policy_id == requirement.source_policy_id + and quality.state in {"LIVE", "STALE"} + and quality.event_recency_state in {"LIVE", "STALE"} + and ( + ( + quality.provider_session_state == "NOT_APPLICABLE" + and quality.provider_session_liveness_ms is None + and requirement.max_session_liveness_ms is None + ) + or ( + quality.provider_session_state == "LIVE" + and quality.provider_session_liveness_ms is not None + and ( + requirement.max_session_liveness_ms is None + or quality.provider_session_liveness_ms + <= requirement.max_session_liveness_ms + ) + ) + ) + and quality.complete + and not quality.gap_open + ) + + async def _wait_for_live_snapshot_retry( client: AsyncDataLayerClient, *, @@ -556,7 +952,7 @@ async def _wait_for_live_snapshot_retry( ) -> None: if error.code != "DATA_STALE": raise error - if product.feed.value not in {"QUOTE", "TRADE"}: + if product.feed.value not in {"QUOTE", "BOOK_SNAPSHOT", *_QUIET_CONTINUITY_FEEDS}: raise error remaining = deadline - time.monotonic() if remaining <= 0: @@ -574,15 +970,45 @@ async def _wait_for_live_snapshot_retry( retryable = _quiet_quote_is_retryable( product, requirement, status ) or _fresh_quote_status_is_retryable(product, requirement, status) + retry_delay_seconds = _QUIET_QUOTE_RETRY_SECONDS + if not retryable: + retryable = _transitional_session_status_is_retryable( + product, requirement, status + ) + retry_delay_seconds = _TRANSIENT_SESSION_RETRY_SECONDS + elif product.feed.value == "BOOK_SNAPSHOT": + retryable = _book_snapshot_status_is_refreshable(product, requirement, status) + retry_delay_seconds = _BOOK_SNAPSHOT_RETRY_SECONDS + elif product.feed.value == "TRADE" and requirement.max_session_liveness_ms is None: + retryable = _fresh_trade_without_session_sla_is_retryable( + product, requirement, status + ) + retry_delay_seconds = _QUIET_QUOTE_RETRY_SECONDS else: retryable = ( - _quiet_trade_status_is_observable(product, requirement, status) - or _fresh_trade_status_is_observable(product, requirement, status) + _quiet_continuity_status_is_observable(product, requirement, status) + or _fresh_continuity_status_is_observable(product, requirement, status) ) + if not retryable: + retryable = _transitional_session_status_is_retryable( + product, requirement, status + ) + retry_delay_seconds = _TRANSIENT_SESSION_RETRY_SECONDS if not retryable: - raise ContinuityError( + required_state = ( + "a verified complete, gap-free snapshot state" + if product.feed.value == "BOOK_SNAPSHOT" + else ( + "a fresh executable status" + if product.feed.value == "TRADE" + and requirement.max_session_liveness_ms is None + else "a live provider session" + ) + ) + raise C2StatusEvidenceError( "DATA_STALE", - f"C2 strict {product.feed.value} retry requires a live provider session", + f"C2 strict {product.feed.value} retry requires {required_state}", + status=status, ) from error remaining = deadline - time.monotonic() if remaining <= 0: @@ -683,92 +1109,120 @@ async def _stream_resume( grpc_target: str, state_dir: Path, timeout_seconds: float, + stream_open_timeout_seconds: float | None = None, + client_factory: Callable[..., AsyncDataLayerClient] | None = None, ) -> tuple[int | None, int | None, tuple[str, ...], tuple[str, ...]]: if product.delivery is not DeliveryClass.DURABLE: return None, None, (), () cursor_path = _cursor_path(state_dir, product) - requirement = sdk_requirement(product) - historical_replay = _uses_historical_bar_replay(product) + requirement = _c2_requirement(sdk_requirement(product)) stream_requirement = requirement - strict_watermark: int | None = None event_timeout_seconds = _stream_event_timeout_seconds(product, timeout_seconds) - quiet_trade_observation = _allows_quiet_trade_observation(product, requirement) + quiet_continuity_observation = _allows_quiet_continuity_observation(product, requirement) + quiet_final_bar_handoff = _allows_quiet_final_bar_handoff(product) quiet_primary = False no_event_sessions: list[str] = [] first_controls: list[str] = [] first_offset: int | None = None - first_client = _client( + first_client = _receipt_client( identity, base_url=primary_url, grpc_target=grpc_target, cursor_path=cursor_path, timeout_seconds=timeout_seconds, + client_factory=client_factory, ) try: - if historical_replay: - # This remains the product's actual contract check. The replay - # cursor below is deliberately older only to make the bounded C2 - # reconnect proof independent of the next 15m/1h close. - strict_warmup = await first_client.warmup(requirement) - strict_current = strict_warmup.data[-1] - validate_product_view(product, strict_current) - strict_watermark = strict_warmup.watermark_offset - stream_requirement = _historical_bar_replay_requirement( - requirement, - latest_open_time_ns=int(strict_current.payload.open_time_ns), - ) - event_timeout_seconds = timeout_seconds async with _strict_warmup_then_stream_for_c2( first_client, product=product, requirement=stream_requirement, timeout_seconds=timeout_seconds, ) as session: - if quiet_trade_observation: + if quiet_continuity_observation or quiet_final_bar_handoff: first, first_controls = await _next_data_or_timeout( session, timeout_seconds=min( event_timeout_seconds, - _QUIET_TRADE_STREAM_OBSERVATION_SECONDS, + _QUIET_CONTINUITY_STREAM_OBSERVATION_SECONDS, ), + stream_open_timeout_seconds=stream_open_timeout_seconds, ) if first is None: _require_signed_cursor_controls(first_controls) - no_event_sessions.append(await _classify_no_event_trade_session( - first_client, - product=product, - requirement=requirement, - timeout_seconds=timeout_seconds, - )) + if quiet_final_bar_handoff: + no_event_sessions.append( + await _verify_quiet_final_bar_current( + first_client, + product=product, + requirement=requirement, + timeout_seconds=timeout_seconds, + ) + ) + else: + no_event_sessions.append(await _classify_no_event_continuity_session( + first_client, + product=product, + requirement=requirement, + timeout_seconds=timeout_seconds, + )) quiet_primary = True else: first, first_controls = await _next_data( session, timeout_seconds=event_timeout_seconds, + stream_open_timeout_seconds=stream_open_timeout_seconds, ) if not quiet_primary: assert first is not None - first_view = market_data_view_from_stream( - first, - template=session.warmup.data[-1], - requirement=stream_requirement, - ) + first_replay_only = False + try: + first_view = market_data_view_from_stream( + first, + template=session.warmup.data[-1], + requirement=stream_requirement, + ) + except ContinuityError as error: + # A snapshot cursor can race a delayed provider frame. The + # frame is never execution input: accept it only as signed + # replay state and immediately re-read strict current V2 + # quality below. Gaps, identity violations and every other + # continuity error remain fail-closed. + if error.code != "DATA_STALE": + raise + first_view = market_data_view_from_stream( + first, + template=session.warmup.data[-1], + requirement=stream_requirement, + replay_only=True, + ) + first_replay_only = True validate_product_view( product, first_view, - require_current_quality=not historical_replay, + require_current_quality=not first_replay_only, + **({"state_replay": True} if first_replay_only else {}), ) session.acknowledge(first) first_offset = first.logical_offset + if first_replay_only: + current = await _strict_snapshot_for_c2( + first_client, + product=product, + requirement=requirement, + timeout_seconds=timeout_seconds, + ) + validate_product_view(product, current.data) finally: await first_client.close() - resumed_client = _client( + resumed_client = _receipt_client( identity, base_url=secondary_url, grpc_target=grpc_target, cursor_path=cursor_path, timeout_seconds=timeout_seconds, + client_factory=client_factory, ) try: if quiet_primary: @@ -786,17 +1240,28 @@ async def _stream_resume( session, timeout_seconds=min( event_timeout_seconds, - _QUIET_TRADE_STREAM_OBSERVATION_SECONDS, + _QUIET_CONTINUITY_STREAM_OBSERVATION_SECONDS, ), + stream_open_timeout_seconds=stream_open_timeout_seconds, ) if observed is None: _require_signed_cursor_controls(observed_controls) - no_event_sessions.append(await _classify_no_event_trade_session( - resumed_client, - product=product, - requirement=requirement, - timeout_seconds=timeout_seconds, - )) + if quiet_final_bar_handoff: + no_event_sessions.append( + await _verify_quiet_final_bar_current( + resumed_client, + product=product, + requirement=requirement, + timeout_seconds=timeout_seconds, + ) + ) + else: + no_event_sessions.append(await _classify_no_event_continuity_session( + resumed_client, + product=product, + requirement=requirement, + timeout_seconds=timeout_seconds, + )) else: observed_view = market_data_view_from_stream( observed, @@ -823,28 +1288,37 @@ async def _stream_resume( ) as session: acknowledged_offset = first_offset resumed_controls: list[str] = [] - maximum = ( - _MAX_HISTORICAL_REPLAY_CATCHUP_EVENTS - if historical_replay - else 1 - ) - for _ in range(maximum): - if quiet_trade_observation: + # A historical seed intentionally starts before the current + # snapshot so C2 can prove signed replay without waiting for the + # next bar close. It must not be treated as a production consumer + # catch-up cursor: authentic late backfills can make its append + # offset far older than the current strict watermark. One + # monotonic replay across replicas proves cursor continuity; the + # strict snapshot below proves current executable quality. + for _ in range(1): + if quiet_continuity_observation or quiet_final_bar_handoff: resumed, controls = await _next_data_or_timeout( session, timeout_seconds=min( event_timeout_seconds, - _QUIET_TRADE_STREAM_OBSERVATION_SECONDS, + _QUIET_CONTINUITY_STREAM_OBSERVATION_SECONDS, ), + stream_open_timeout_seconds=stream_open_timeout_seconds, ) if resumed is None: _require_signed_cursor_controls(controls) - no_event_session = await _classify_no_event_trade_session( - resumed_client, - product=product, - requirement=requirement, - timeout_seconds=timeout_seconds, - ) + if quiet_final_bar_handoff: + no_event_session = await _verify_quiet_final_bar_current( + resumed_client, product=product, requirement=requirement, + timeout_seconds=timeout_seconds, + ) + else: + no_event_session = await _classify_no_event_continuity_session( + resumed_client, + product=product, + requirement=requirement, + timeout_seconds=timeout_seconds, + ) return ( None, None, @@ -855,17 +1329,26 @@ async def _stream_resume( resumed, controls = await _next_data( session, timeout_seconds=event_timeout_seconds, + stream_open_timeout_seconds=stream_open_timeout_seconds, ) resumed_controls.extend(controls) + # This context was opened with `resume_restored_state=True`. + # Its bounded frame proves cursor recovery only, even when the + # durable offset is newer than the reconnect snapshot watermark. + # A strict V2 snapshot below is the sole current/executable + # attestation after reconnect. + replay_only = True resumed_view = market_data_view_from_stream( resumed, template=session.warmup.data[-1], requirement=stream_requirement, + **({"replay_only": True} if replay_only else {}), ) validate_product_view( product, resumed_view, - require_current_quality=not historical_replay, + require_current_quality=not replay_only, + **({"state_replay": True} if replay_only else {}), ) validate_resume_offsets( acknowledged_offset=acknowledged_offset, @@ -873,19 +1356,24 @@ async def _stream_resume( ) session.acknowledge(resumed) acknowledged_offset = resumed.logical_offset - if ( - strict_watermark is None - or acknowledged_offset >= strict_watermark - ): - return ( - first_offset, - acknowledged_offset, - tuple(first_controls + resumed_controls), - (), + if replay_only: + # The replay frame is state recovery only. A fresh strict + # V2 read is required before C2 can attest current quality, + # including after a stale first stream frame. + current = await _strict_snapshot_for_c2( + resumed_client, + product=product, + requirement=requirement, + timeout_seconds=timeout_seconds, ) - raise AssertionError( - "historical BAR replay did not converge through the strict current watermark" - ) + validate_product_view(product, current.data) + return ( + first_offset, + acknowledged_offset, + tuple(first_controls + resumed_controls), + (), + ) + raise AssertionError("signed cursor replay did not emit a data event") finally: await resumed_client.close() @@ -899,20 +1387,24 @@ async def _certify_product( grpc_target: str, state_dir: Path, timeout_seconds: float, + stream_open_timeout_seconds: float | None = None, + client_factory: Callable[..., AsyncDataLayerClient] | None = None, ) -> dict[str, object]: - primary = _client( + primary = _receipt_client( identity, base_url=primary_url, grpc_target=grpc_target, cursor_path=state_dir / "query-primary.json", timeout_seconds=timeout_seconds, + client_factory=client_factory, ) - secondary = _client( + secondary = _receipt_client( identity, base_url=secondary_url, grpc_target=grpc_target, cursor_path=state_dir / "query-secondary.json", timeout_seconds=timeout_seconds, + client_factory=client_factory, ) try: ( @@ -939,6 +1431,8 @@ async def _certify_product( grpc_target=grpc_target, state_dir=state_dir, timeout_seconds=timeout_seconds, + stream_open_timeout_seconds=stream_open_timeout_seconds, + client_factory=client_factory, ) result = compact_receipt_evidence( product, diff --git a/scripts/phase105_consumer_v2_identity_acceptance.py b/scripts/phase105_consumer_v2_identity_acceptance.py index 91885093..13c94134 100644 --- a/scripts/phase105_consumer_v2_identity_acceptance.py +++ b/scripts/phase105_consumer_v2_identity_acceptance.py @@ -10,16 +10,20 @@ import argparse import asyncio +import hashlib import json import resource import shutil import sys import tempfile import time -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path +from typing import Callable from urllib.parse import urlsplit +import httpx + ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) @@ -27,6 +31,12 @@ from qdl.certification.phase103_consumer_acceptance import ( AcceptanceProduct, DeliveryClass, + content_fingerprint, + sdk_requirement, + validate_final_bar_warmup_windows, + validate_product_view, + validate_replica_views, + warmup_content_fingerprint, ) from qdl.certification.phase105_consumer_acceptance import ( PHASE105_PAPER_CONSUMER_IDS, @@ -35,6 +45,7 @@ from qdl.certification.reference_l2_acceptance import ( ReferenceAcceptanceProduct, acceptance_transport_timeout_seconds, + is_rust_admitted_native_basis, reference_acceptance_batches, reference_evidence, reference_quality, @@ -50,17 +61,22 @@ validate_v1_runtime_binding, ) from qdl.consumer import StableReleaseRoutePlan, requirement_key +from qdl.certification.phase105_release_observations import compact_view_quality from qdl.runtime.stable_catalog import StableSourceCatalog from qdl.runtime.stable_deployment import StableAcquisitionPlan from scripts.phase103_consumer_receipt_acceptance import ( + C2StatusEvidenceError, + _c2_requirement, _certify_product, _client, + compact_feed_status, _identity, _query_product, ) from scripts.phasec36_reference_l2_consumer_acceptance import ( _reference_batch_until_terminal, ) +from qdl_sdk.errors import DataLayerError IDENTITY_PREFIXES = { @@ -78,6 +94,10 @@ "qdl-v2-stream-b:8210", }) _MAX_REFERENCE_BATCH_CONCURRENCY = 4 +_C2_REQUEST_QUOTA_FRACTION = 0.75 +_C2_QUOTA_WINDOW_MARGIN_SECONDS = 0.05 +_C2_OPENING_TIMEOUT_SECONDS = 900.0 +_C2_CLOSING_REVALIDATION_MAX_SECONDS = 120.0 @dataclass(frozen=True, slots=True) @@ -88,6 +108,252 @@ class IdentityFiles: jwt_key_id: str +class C2ProductAcceptanceError(RuntimeError): + """One compact, payload-free product failure for an operator C2 receipt.""" + + def __init__(self, product: AcceptanceProduct, error: C2StatusEvidenceError) -> None: + super().__init__( + "Phase 10.5 V2 identity receipt failed " + f"consumer={product.consumer_id} instrument={product.instrument_id} " + f"feed={product.feed.value} interval={product.interval}" + ) + self.evidence = { + "schema": "qdl.phase105.c2-product-failure.v1", + "product": product.evidence(), + "replica": error.replica or "unknown", + "error_code": error.code, + "typed_status": error.status_evidence, + "payload_recorded": False, + } + + +class C2ClosingBatchError(RuntimeError): + """Compact, payload-free evidence for a closing batch transport failure.""" + + def __init__( + self, + *, + consumer_id: str, + replica: str, + products: tuple[AcceptanceProduct, ...], + error: Exception, + status_observations: list[dict[str, object]], + ) -> None: + if not products or any(item.consumer_id != consumer_id for item in products): + raise ValueError("Phase 10.5 closing batch failure has an invalid consumer scope") + digest = hashlib.sha256( + json.dumps( + [item.identity for item in products], + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() + super().__init__( + "Phase 10.5 V2 closing batch failed " + f"consumer={consumer_id} replica={replica} size={len(products)}" + ) + self.evidence = { + "schema": "qdl.phase105.c2-closing-batch-failure.v1", + "consumer_id": consumer_id, + "replica": replica, + "batch_size": len(products), + "batch_identity_sha256": digest, + "transport_error": type(error).__name__, + "typed_status": status_observations, + "payload_recorded": False, + } + + +class _C2ConsumerRequestPacer: + """Keep the disposable C2 probe below one manifest's real REST quota. + + The stable data-plane limit is enforced by Redis in wall-clock minute + buckets. C2 shares an identity across both query replicas, so one local + pacer must serialize every REST request for that identity. It deliberately + uses only 75% of the sealed quota, leaving headroom for an independently + running paper consumer without changing its production allowance. + """ + + def __init__( + self, + requests_per_minute: int, + *, + safety_fraction: float = _C2_REQUEST_QUOTA_FRACTION, + clock: Callable[[], float] = time.time, + sleep: Callable[[float], object] | None = None, + ) -> None: + if requests_per_minute < 1: + raise ValueError("C2 manifest request quota must be positive") + if not 0.0 < safety_fraction < 1.0: + raise ValueError("C2 request quota fraction must be between zero and one") + self.requests_per_minute = requests_per_minute + self.safe_requests_per_minute = max( + 1, int(requests_per_minute * safety_fraction) + ) + self._seconds_per_request = 60.0 / self.safe_requests_per_minute + self._clock = clock + self._sleep = sleep + self._lock = asyncio.Lock() + self._next_at: float | None = None + self._request_count = 0 + self._wait_seconds = 0.0 + self._window_wait_seconds = 0.0 + + async def wait_for_clean_window(self) -> float: + """Start C2 only after the next shared Redis quota-minute boundary.""" + + now = self._clock() + target = (int(now // 60.0) + 1) * 60.0 + _C2_QUOTA_WINDOW_MARGIN_SECONDS + wait_seconds = max(0.0, target - now) + sleeper = asyncio.sleep if self._sleep is None else self._sleep + if wait_seconds > 0: + await sleeper(wait_seconds) + async with self._lock: + self._next_at = self._clock() + self._window_wait_seconds += wait_seconds + return wait_seconds + + async def acquire(self) -> None: + """Reserve one real REST request without borrowing quota from a peer.""" + + async with self._lock: + now = self._clock() + target = now if self._next_at is None else max(now, self._next_at) + self._next_at = target + self._seconds_per_request + self._request_count += 1 + wait_seconds = max(0.0, target - now) + self._wait_seconds += wait_seconds + sleeper = asyncio.sleep if self._sleep is None else self._sleep + if wait_seconds > 0: + await sleeper(wait_seconds) + + def evidence(self) -> dict[str, object]: + return { + "requests_per_minute": self.requests_per_minute, + "c2_safe_requests_per_minute": self.safe_requests_per_minute, + "c2_request_count": self._request_count, + "c2_pacing_wait_seconds": round(self._wait_seconds, 3), + "c2_clean_window_wait_seconds": round(self._window_wait_seconds, 3), + } + + +class _PacedQueryTransport: + """Acceptance-only adapter that charges every C2 REST call to one pacer.""" + + def __init__(self, delegate, pacer: _C2ConsumerRequestPacer) -> None: + self._delegate = delegate + self._pacer = pacer + + async def _call(self, name: str, *args, **kwargs): + await self._pacer.acquire() + return await getattr(self._delegate, name)(*args, **kwargs) + + async def warmup(self, *args, **kwargs): + return await self._call("warmup", *args, **kwargs) + + async def warmup_batch(self, *args, **kwargs): + return await self._call("warmup_batch", *args, **kwargs) + + async def reference_batch(self, *args, **kwargs): + return await self._call("reference_batch", *args, **kwargs) + + async def snapshot(self, *args, **kwargs): + return await self._call("snapshot", *args, **kwargs) + + async def feed_status(self, *args, **kwargs): + return await self._call("feed_status", *args, **kwargs) + + async def instruments(self, *args, **kwargs): + return await self._call("instruments", *args, **kwargs) + + async def instrument(self, *args, **kwargs): + return await self._call("instrument", *args, **kwargs) + + async def close(self) -> None: + await self._delegate.close() + + +class _PacedStreamTransport: + """Charge each C2 stream open to the same per-identity request budget.""" + + def __init__(self, delegate, pacer: _C2ConsumerRequestPacer) -> None: + self._delegate = delegate + self._pacer = pacer + + async def subscribe(self, *args, **kwargs): + # `subscribe` is an async iterator. Reserving at iterator start covers + # both the initial stream and every SDK reconnect without changing the + # public stream contract. + await self._pacer.acquire() + async for item in self._delegate.subscribe(*args, **kwargs): + yield item + + async def close(self) -> None: + await self._delegate.close() + + +def _paced_client_factory(pacer: _C2ConsumerRequestPacer): + """Preserve the SDK contract while pacing C2 REST and stream opens only.""" + + def create(identity, *, base_url, grpc_target, cursor_path, timeout_seconds): + client = _client( + identity, + base_url=base_url, + grpc_target=grpc_target, + cursor_path=cursor_path, + timeout_seconds=timeout_seconds, + ) + client.query_transport = _PacedQueryTransport(client.query_transport, pacer) + client.stream_transport = _PacedStreamTransport(client.stream_transport, pacer) + return client + + return create + + +def _quota_pacers( + release: StableReleaseRoutePlan, + consumer_ids: tuple[str, ...], +) -> dict[str, _C2ConsumerRequestPacer]: + routes = {item.consumer_id: item for item in release.consumers} + if any(consumer_id not in routes for consumer_id in consumer_ids): + raise ValueError("Phase 10.5 C2 consumer quota manifest is unavailable") + return { + consumer_id: _C2ConsumerRequestPacer( + routes[consumer_id].manifest.quotas.requests_per_minute + ) + for consumer_id in consumer_ids + } + + +async def _wait_for_clean_quota_windows( + pacers: dict[str, _C2ConsumerRequestPacer], +) -> float: + """Align all governed identities with a fresh server-side quota minute.""" + + if not pacers: + raise ValueError("Phase 10.5 C2 requires at least one quota pacer") + waits = await asyncio.gather(*(item.wait_for_clean_window() for item in pacers.values())) + return max(waits) + + +async def _wait_for_minimum_observation( + *, + started_monotonic: float, + observation_seconds: float, +) -> float: + """Hold a real C2 observation window before full closing revalidation.""" + + if observation_seconds <= 0: + raise ValueError("C2 observation duration must be positive") + remaining = started_monotonic + observation_seconds - time.monotonic() + if remaining > 0: + await asyncio.sleep(remaining) + elapsed = time.monotonic() - started_monotonic + if elapsed + 0.001 < observation_seconds: + raise AssertionError("Phase 10.5 C2 observation ended before its declared duration") + return elapsed + + def _authority(path: Path) -> dict[str, object]: try: value = json.loads(path.read_text(encoding="utf-8")) @@ -108,6 +374,28 @@ def _reference_batch_concurrency(observation_concurrency: int) -> int: return min(observation_concurrency, _MAX_REFERENCE_BATCH_CONCURRENCY) +def _reference_transport_timeout_seconds( + products: tuple[ReferenceAcceptanceProduct, ...], + *, + generic_timeout_seconds: float, +) -> float: + """Keep the disposable client alive through its declared provider contract. + + Durable query/stream reads retain the generic C2 timeout. Reference + batches carry their own bounded provider deadline, so a client must not + cancel a valid request before that contract plus the response margin. + """ + + if not products: + raise ValueError("C2 reference transport requires at least one product") + declared_timeout_seconds = max( + item.sdk_requirement.deadline_ms / 1_000 for item in products + ) + return acceptance_transport_timeout_seconds( + max(generic_timeout_seconds, declared_timeout_seconds) + ) + + def _identity_files(args: argparse.Namespace) -> dict[str, IdentityFiles]: return _identity_files_for_consumers(args, tuple(IDENTITY_PREFIXES)) @@ -227,6 +515,8 @@ async def _certify_references( timeout_seconds: float, deadline_monotonic: float, semaphore: asyncio.Semaphore, + native_basis_semaphore: asyncio.Semaphore, + client_factory, ) -> list[dict[str, object]]: """Read declared provider data through both V2 replicas, never V1/direct. @@ -241,20 +531,24 @@ async def _certify_references( ) if not reference_products: return [] - transport_timeout_seconds = acceptance_transport_timeout_seconds(timeout_seconds) + transport_timeout_seconds = _reference_transport_timeout_seconds( + reference_products, + generic_timeout_seconds=timeout_seconds, + ) async def read_replica(client, *, label: str): values: dict[tuple[str, str, str, str, str], tuple[str, float, int, int, dict[str, int | bool]]] = {} try: for batch in reference_acceptance_batches(reference_products): - async with semaphore: - started = time.perf_counter() - response, attempts, deferred_ms = await _reference_batch_until_terminal( - client, - batch, - deadline_monotonic=deadline_monotonic, - ) - latency_ms = (time.perf_counter() - started) * 1_000 + started = time.perf_counter() + response, attempts, deferred_ms = await _reference_batch_for_c2( + client, + batch, + deadline_monotonic=deadline_monotonic, + semaphore=semaphore, + native_basis_semaphore=native_basis_semaphore, + ) + latency_ms = (time.perf_counter() - started) * 1_000 observed_at_ns = time.time_ns() hashes = tuple( reference_evidence(item, result, observed_at_ns=observed_at_ns) @@ -276,14 +570,14 @@ async def read_replica(client, *, label: str): raise AssertionError(f"Phase 10.5 {label} reference batch lost a product") return values - primary = _client( + primary = client_factory( identity, base_url=primary_url, grpc_target=grpc_target, cursor_path=state_dir / "reference-primary.json", timeout_seconds=transport_timeout_seconds, ) - secondary = _client( + secondary = client_factory( identity, base_url=secondary_url, grpc_target=grpc_target, @@ -354,15 +648,16 @@ async def _v2_query_product( grpc_target: str, state_dir: Path, timeout_seconds: float, + client_factory, ) -> tuple[str, str | None, float, float | None]: - primary = _client( + primary = client_factory( identity, base_url=primary_url, grpc_target=grpc_target, cursor_path=state_dir / "fallback-query-primary.json", timeout_seconds=timeout_seconds, ) - secondary = _client( + secondary = client_factory( identity, base_url=secondary_url, grpc_target=grpc_target, @@ -387,6 +682,7 @@ async def _v1_fallback_return( v1_base_url: str, state_dir: Path, timeout_seconds: float, + client_factory, ) -> dict[str, object]: """Read V2, make one allowed V1 cached read, then confirm V2 again.""" before = await _v2_query_product( @@ -397,6 +693,7 @@ async def _v1_fallback_return( grpc_target=grpc_target, state_dir=state_dir / "before", timeout_seconds=timeout_seconds, + client_factory=client_factory, ) import httpx @@ -419,6 +716,7 @@ async def _v1_fallback_return( grpc_target=grpc_target, state_dir=state_dir / "after", timeout_seconds=timeout_seconds, + client_factory=client_factory, ) return { **details, @@ -430,6 +728,299 @@ async def _v1_fallback_return( } +def _chunks(values: tuple[AcceptanceProduct, ...], size: int): + if size < 1: + raise ValueError("Phase 10.5 C2 batch size must be positive") + for offset in range(0, len(values), size): + yield values[offset:offset + size] + + +def _closing_batches(products: tuple[AcceptanceProduct, ...], max_batch_items: int): + """Keep hot reads out of history batches and bound head-of-line delay.""" + groups: dict[str, list[AcceptanceProduct]] = {} + for product in products: + groups.setdefault(product.feed.value, []).append(product) + for feed, group in groups.items(): + size = max_batch_items if feed == "BAR" else min(max_batch_items, 8) + yield from _chunks(tuple(group), size) + + +def _closing_requirement(product: AcceptanceProduct): + """Keep closing current-state proof small without weakening the product. + + Opening C2 already proves the declared bounded BAR history, finality and + signed stream handoff. Closing keeps up to two final BARs so reads spanning one + candle close still have an immutable overlap. Each current tail remains + strict, and the existing parity validator rejects larger window shifts. + Every non-history policy field is retained unchanged. + """ + + requirement = _c2_requirement(sdk_requirement(product)) + if requirement.feed.value != "BAR": + return requirement + specification = requirement.warmup_specification + if specification is None or specification.rows is None: + raise ValueError("Phase 10.5 closing BAR requires a row-bounded warmup policy") + rows = min(2, specification.rows) + return replace( + requirement, + warmup_limit=rows, + warmup=( + requirement.warmup.model_copy(update={"rows": rows}) + if requirement.warmup is not None + else None + ), + ) + + +def _closing_status_representatives( + products: tuple[AcceptanceProduct, ...], +) -> tuple[AcceptanceProduct, ...]: + """Keep transport-failure evidence bounded to one identity per feed.""" + + by_feed: dict[str, AcceptanceProduct] = {} + for product in products: + by_feed.setdefault(product.feed.value, product) + return tuple(by_feed[feed] for feed in sorted(by_feed)) + + +async def _closing_failure_status_observations( + client, + products: tuple[AcceptanceProduct, ...], + *, + timeout_seconds: float, +) -> list[dict[str, object]]: + """Capture bounded typed status after a failed closing batch, never payload.""" + + timeout = min(5.0, timeout_seconds) + observations: list[dict[str, object]] = [] + for product in _closing_status_representatives(products): + try: + status = await asyncio.wait_for( + client.feed_status(_closing_requirement(product)), + timeout=timeout, + ) + except Exception as error: # Diagnostic must not hide the primary failure. + observations.append({ + **product.evidence(), + "status_transport_error": type(error).__name__, + }) + else: + observations.append({ + **product.evidence(), + "quality": compact_feed_status(status), + }) + return observations + + +async def _closing_batch_revalidation( + products: tuple[AcceptanceProduct, ...], + *, + identity, + primary_url: str, + secondary_url: str, + grpc_target: str, + state_dir: Path, + timeout_seconds: float, + max_batch_items: int, + client_factory, +) -> list[dict[str, object]]: + """Re-read every durable/pass-through product through both V2 replicas. + + C2's opening proof already establishes signed cursor/reconnect per product. + Closing needs a strict current view for every route, not a second identical + stream storm. `warmup:batch` keeps that full-scope check below the real + per-identity request quota without weakening any product validation. + """ + + if not products: + return [] + if not 1 <= max_batch_items <= 100: + raise ValueError("Phase 10.5 C2 batch size exceeds the V2 contract") + + async def read_replica(base_url: str, *, label: str): + client = client_factory( + identity, + base_url=base_url, + grpc_target=grpc_target, + cursor_path=state_dir / f"closing-{label}.json", + timeout_seconds=timeout_seconds, + ) + values: dict[tuple[str, str, str, str, str], dict[str, object]] = {} + try: + for batch in _closing_batches(products, max_batch_items): + requirements = tuple(_closing_requirement(item) for item in batch) + started = time.perf_counter() + try: + response = await client.warmup_batch(requirements, require_all=True) + except (httpx.HTTPError, TimeoutError, DataLayerError) as error: + status_observations = await _closing_failure_status_observations( + client, + batch, + timeout_seconds=timeout_seconds, + ) + raise C2ClosingBatchError( + consumer_id=batch[0].consumer_id, + replica=label, + products=batch, + error=error, + status_observations=status_observations, + ) from error + latency_ms = (time.perf_counter() - started) * 1_000 + if response.partial or len(response.results) != len(batch): + raise AssertionError("Phase 10.5 closing V2 batch cardinality differs") + observed_at_ns = time.time_ns() + for product, item in zip(batch, response.results, strict=True): + if item.data is None or not item.data.data: + raise AssertionError("Phase 10.5 closing V2 batch returned no product data") + history = tuple(item.data.data) + for view in history[:-1]: + validate_product_view( + product, view, require_current_quality=False + ) + latest = history[-1] + validate_product_view(product, latest) + if product.identity in values: + raise AssertionError("Phase 10.5 closing V2 batch duplicated a product") + values[product.identity] = { + "history": history, + "latest": latest, + "latency_ms": latency_ms, + "quality": compact_view_quality( + latest, observed_at_ns=observed_at_ns + ), + } + finally: + await client.close() + if len(values) != len(products): + raise AssertionError("Phase 10.5 closing V2 batch lost a product") + return values + + primary_values, secondary_values = await asyncio.gather( + read_replica(primary_url, label="primary"), + read_replica(secondary_url, label="secondary"), + ) + evidence: list[dict[str, object]] = [] + for product in products: + primary = primary_values[product.identity] + secondary = secondary_values[product.identity] + bar_alignment: dict[str, object] | None = None + if product.feed.value == "BAR": + primary_hash = warmup_content_fingerprint(primary["history"]) + secondary_hash = warmup_content_fingerprint(secondary["history"]) + if product.delivery is DeliveryClass.DURABLE: + bar_alignment = validate_final_bar_warmup_windows( + primary["history"], secondary["history"] + ) + primary_hash = str(bar_alignment["primary_content_sha256"]) + secondary_hash = str(bar_alignment["secondary_content_sha256"]) + else: + validate_replica_views(product, primary["latest"], secondary["latest"]) + else: + primary_hash, secondary_hash = validate_replica_views( + product, primary["latest"], secondary["latest"] + ) + item_evidence = { + **product.evidence(), + "primary_content_sha256": primary_hash, + "secondary_content_sha256": secondary_hash, + "primary_latency_ms": round(float(primary["latency_ms"]), 3), + "secondary_latency_ms": round(float(secondary["latency_ms"]), 3), + "release_quality": { + "primary": primary["quality"], + "secondary": secondary["quality"], + }, + "closing_read": "BATCH_V2_PRIMARY", + } + if bar_alignment is not None: + item_evidence["bar_replica_alignment"] = bar_alignment + evidence.append(item_evidence) + return evidence + + +async def _reference_batch_for_c2( + client, + batch: tuple[ReferenceAcceptanceProduct, ...], + *, + deadline_monotonic: float, + semaphore: asyncio.Semaphore, + native_basis_semaphore: asyncio.Semaphore, +): + """Respect Rust's one native-BASIS lane across all C2 identities/replicas.""" + + native_basis = tuple(item for item in batch if is_rust_admitted_native_basis(item)) + if native_basis and len(native_basis) != len(batch): + raise AssertionError("Phase 10.5 native BASIS batch mixes provider lanes") + if native_basis: + if len(native_basis) != 1: + raise AssertionError("Phase 10.5 native BASIS batch must be singleton") + async with native_basis_semaphore: + async with semaphore: + return await _reference_batch_until_terminal( + client, + batch, + deadline_monotonic=deadline_monotonic, + ) + async with semaphore: + return await _reference_batch_until_terminal( + client, + batch, + deadline_monotonic=deadline_monotonic, + ) + + +async def _closing_revalidate_consumer( + consumer_id: str, + products: tuple[AcceptanceProduct, ...], + *, + identity, + primary_url: str, + secondary_url: str, + grpc_target: str, + state_dir: Path, + timeout_seconds: float, + deadline_monotonic: float, + max_batch_items: int, + reference_semaphore: asyncio.Semaphore, + native_basis_semaphore: asyncio.Semaphore, + client_factory, +) -> list[dict[str, object]]: + stream_products = tuple( + item for item in products if item.delivery is not DeliveryClass.ON_DEMAND + ) + reference_products = tuple( + item for item in products if item.delivery is DeliveryClass.ON_DEMAND + ) + stream_task = asyncio.create_task(_closing_batch_revalidation( + stream_products, + identity=identity, + primary_url=primary_url, + secondary_url=secondary_url, + grpc_target=grpc_target, + state_dir=state_dir / "stream", + timeout_seconds=timeout_seconds, + max_batch_items=max_batch_items, + client_factory=client_factory, + )) + reference_task = asyncio.create_task(_certify_references( + reference_products, + identity=identity, + primary_url=primary_url, + secondary_url=secondary_url, + grpc_target=grpc_target, + state_dir=state_dir / "references", + timeout_seconds=timeout_seconds, + deadline_monotonic=deadline_monotonic, + semaphore=reference_semaphore, + native_basis_semaphore=native_basis_semaphore, + client_factory=client_factory, + )) + stream_results, reference_results = await _gather_or_cancel((stream_task, reference_task)) + if len(stream_results) != len(stream_products) or len(reference_results) != len(reference_products): + raise AssertionError("Phase 10.5 closing V2 scope cardinality differs") + return [*stream_results, *reference_results] + + async def _run_consumer_groups( consumer_ids: tuple[str, ...], run_group, @@ -502,13 +1093,22 @@ async def run(args: argparse.Namespace) -> dict[str, object]: if not isinstance(identities[product.consumer_id].tls, WorkloadTlsConfig): raise AssertionError("Phase 10.5 identity did not build workload TLS") - started = time.monotonic() process_started = time.process_time() temporary = Path(tempfile.mkdtemp(prefix="qdl-phase105-v2-identity-")) product_semaphore = asyncio.Semaphore(args.concurrency) reference_semaphore = asyncio.Semaphore( _reference_batch_concurrency(args.concurrency) ) + native_basis_semaphore = asyncio.Semaphore(1) + pacers = _quota_pacers(release, consumer_ids) + client_factories = { + consumer_id: _paced_client_factory(pacer) + for consumer_id, pacer in pacers.items() + } + release_consumers = {item.consumer_id: item for item in release.consumers} + quota_window_wait_seconds = await _wait_for_clean_quota_windows(pacers) + started = time.monotonic() + opening_deadline = started + args.opening_timeout_seconds async def certify(product: AcceptanceProduct) -> dict[str, object]: async with product_semaphore: @@ -521,7 +1121,15 @@ async def certify(product: AcceptanceProduct) -> dict[str, object]: grpc_target=grpc_target, state_dir=temporary, timeout_seconds=args.timeout_seconds, + stream_open_timeout_seconds=args.opening_timeout_seconds, + client_factory=client_factories[product.consumer_id], ) + except C2StatusEvidenceError as error: + raise C2ProductAcceptanceError(product, error) from error + except asyncio.CancelledError: + print(json.dumps({"stage": "C2_ACTIVE_PRODUCT_CANCELLED", + "identity": product.identity}), file=sys.stderr, flush=True) + raise except Exception as error: raise RuntimeError( "Phase 10.5 V2 identity receipt failed " @@ -557,8 +1165,10 @@ async def certify_consumer( grpc_target=grpc_target, state_dir=temporary / consumer_id.replace(".", "-") / "references", timeout_seconds=args.timeout_seconds, - deadline_monotonic=started + args.observation_seconds, + deadline_monotonic=opening_deadline, semaphore=reference_semaphore, + native_basis_semaphore=native_basis_semaphore, + client_factory=client_factories[consumer_id], )) task_results = await _gather_or_cancel((*product_tasks, reference_task)) ordered = tuple(task_results[:len(product_tasks)]) @@ -578,6 +1188,7 @@ async def certify_consumer( v1_base_url=v1_base_url, state_dir=temporary / consumer_id.replace(".", "-"), timeout_seconds=args.timeout_seconds, + client_factory=client_factories[consumer_id], )) return [*ordered, *reference_results], fallback_details @@ -592,18 +1203,94 @@ async def certify_ordered() -> tuple[list[dict[str, object]], list[dict[str, obj fallback_details.extend(consumer_fallbacks) return results, fallback_details + async def closing_revalidation_ordered(deadline_monotonic: float) -> list[dict[str, object]]: + async def revalidate_consumer(consumer_id: str) -> list[dict[str, object]]: + consumer_products = tuple( + item for item in scope.products if item.consumer_id == consumer_id + ) + route = release_consumers.get(consumer_id) + if route is None: + raise ValueError("Phase 10.5 closing consumer route is unavailable") + return await _closing_revalidate_consumer( + consumer_id, + consumer_products, + identity=identities[consumer_id], + primary_url=args.primary_url, + secondary_url=args.secondary_url, + grpc_target=grpc_target, + state_dir=temporary / consumer_id.replace(".", "-") / "closing", + timeout_seconds=args.timeout_seconds, + deadline_monotonic=deadline_monotonic, + max_batch_items=route.manifest.quotas.max_batch_items, + reference_semaphore=reference_semaphore, + native_basis_semaphore=native_basis_semaphore, + client_factory=client_factories[consumer_id], + ) + + groups = await _gather_or_cancel(tuple( + asyncio.create_task(revalidate_consumer(consumer_id)) + for consumer_id in consumer_ids + )) + results: list[dict[str, object]] = [] + for consumer_results in groups: + results.extend(consumer_results) + return results + try: - results, fallback_details = await asyncio.wait_for( - certify_ordered(), timeout=args.observation_seconds + # Opening proves warmup/cursor/reconnect/fallback for every product. + # The clock for the true 300-second observation starts only after that + # full proof is complete. Closing rechecks every route with batch V2 + # reads; it deliberately does not create a second stream storm. + opening_started = time.monotonic() + initial_results, initial_fallback_details = await asyncio.wait_for( + certify_ordered(), timeout=args.opening_timeout_seconds + ) + opening_seconds = time.monotonic() - opening_started + print(json.dumps({"stage": "C2_OPENING_PASS", "products": len(initial_results), + "seconds": round(opening_seconds, 3)}), file=sys.stderr, flush=True) + observation_started = time.monotonic() + observation_seconds = await _wait_for_minimum_observation( + started_monotonic=observation_started, + observation_seconds=args.observation_seconds, + ) + closing_started = time.monotonic() + print(json.dumps({"stage": "C2_OBSERVATION_COMPLETE", "seconds": observation_seconds}), + file=sys.stderr, flush=True) + closing_results = await asyncio.wait_for( + closing_revalidation_ordered(closing_started + args.closing_timeout_seconds), + timeout=args.closing_timeout_seconds, ) + closing_seconds = time.monotonic() - closing_started finally: shutil.rmtree(temporary, ignore_errors=True) elapsed_seconds = time.monotonic() - started cpu_seconds = max(0.0, time.process_time() - process_started) max_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss rss_bytes = int(max_rss) * 1024 - if elapsed_seconds > args.observation_seconds: - raise AssertionError("Phase 10.5 identity observation exceeded its bounded window") + if elapsed_seconds > ( + args.opening_timeout_seconds + + args.observation_seconds + + args.closing_timeout_seconds + ): + raise AssertionError("Phase 10.5 identity acceptance exceeded its bounded windows") + initial_by_identity = { + ( + item["consumer_id"], item["instrument_uid"], item["feed"], + item["interval"] or "", item["source_policy_id"], + ): item + for item in initial_results + } + closing_by_identity = { + ( + item["consumer_id"], item["instrument_uid"], item["feed"], + item["interval"] or "", item["source_policy_id"], + ): item + for item in closing_results + } + if set(initial_by_identity) != set(closing_by_identity) or len(initial_by_identity) != len(scope.products): + raise AssertionError("Phase 10.5 identity scope changed during the observation window") + for identity_key, item in initial_by_identity.items(): + item["closing_v2_read"] = closing_by_identity[identity_key] route_summary = _route_summary(release, scope.products) return { "schema": "qdl.phase105.v2-identity-acceptance.v1", @@ -611,11 +1298,11 @@ async def certify_ordered() -> tuple[list[dict[str, object]], list[dict[str, obj "release_route_plan_sha256": release.digest, "authority_revision": authority.get("revision"), "scope_sha256": scope.sha256, - "product_count": len(results), + "product_count": len(initial_results), "durable_product_count": sum( item.delivery is DeliveryClass.DURABLE for item in scope.products ), - "products": results, + "products": initial_results, "route_contract": { **route_summary, "v1_fallback_observed": True, @@ -627,13 +1314,24 @@ async def certify_ordered() -> tuple[list[dict[str, object]], list[dict[str, obj }, "v1_provenance": v1_provenance, "v1_runtime_binding": v1_runtime_binding, - "fallback_details": fallback_details, + "fallback_details": initial_fallback_details, "fallback_drill": build_fallback_return_receipt( release, probes, consumer_ids=consumer_ids ), "provider_connections": 0, "order_actions": 0, "cursor_directory_removed": True, + "observation_seconds_requested": args.observation_seconds, + "observation_seconds_actual": round(observation_seconds, 3), + "opening_product_count": len(initial_results), + "closing_product_count": len(closing_results), + "opening_seconds_actual": round(opening_seconds, 3), + "closing_seconds_actual": round(closing_seconds, 3), + "quota_window_wait_seconds": round(quota_window_wait_seconds, 3), + "quota_budget": { + consumer_id: pacer.evidence() + for consumer_id, pacer in sorted(pacers.items()) + }, "secret_values_recorded": False, "test_provenance": False, "elapsed_seconds": round(elapsed_seconds, 3), @@ -674,6 +1372,15 @@ def parser() -> argparse.ArgumentParser: value.add_argument("--timeout-seconds", type=float, default=15.0) value.add_argument("--concurrency", type=int, default=4) value.add_argument("--observation-seconds", type=float, default=300.0) + value.add_argument( + "--opening-timeout-seconds", type=float, default=_C2_OPENING_TIMEOUT_SECONDS, + help="Bound for the full quota-paced opening proof before observation starts.", + ) + value.add_argument( + "--closing-timeout-seconds", type=float, + default=_C2_CLOSING_REVALIDATION_MAX_SECONDS, + help="Bound for the full-scope batch V2 closing revalidation.", + ) return value @@ -685,7 +1392,22 @@ def main() -> int: raise SystemExit("--concurrency must be between 1 and 8") if not 30.0 <= args.observation_seconds <= 300.0: raise SystemExit("--observation-seconds must be between 30 and 300") - print(json.dumps(asyncio.run(run(args)), sort_keys=True, separators=(",", ":"))) + if not 60.0 <= args.opening_timeout_seconds <= 1_800.0: + raise SystemExit("--opening-timeout-seconds must be between 60 and 1800") + if not 30.0 <= args.closing_timeout_seconds <= 300.0: + raise SystemExit("--closing-timeout-seconds must be between 30 and 300") + try: + result = asyncio.run(run(args)) + except (C2ProductAcceptanceError, C2ClosingBatchError) as error: + print(json.dumps({ + "schema": "qdl.phase105.v2-identity-acceptance.v1", + "status": "FAIL_TYPED_STATUS", + "failure": error.evidence, + "order_actions": 0, + "payload_recorded": False, + }, sort_keys=True, separators=(",", ":"))) + return 1 + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) return 0 diff --git a/scripts/phase105_execution_l2_status_matrix.py b/scripts/phase105_execution_l2_status_matrix.py new file mode 100644 index 00000000..6ffc2c5e --- /dev/null +++ b/scripts/phase105_execution_l2_status_matrix.py @@ -0,0 +1,358 @@ +#!/usr/bin/env python3 +"""Read only the active execution-L2 books through both V2 query replicas. + +This is a preflight, not an ingest or execution client. It derives the exact +physical book scope from the execution demand document, reads typed status and +one public V2 snapshot from each query replica, then writes compact evidence +without retaining market levels, prices, credentials or cursors. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, Mapping + + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from qdl.certification.phase103_consumer_acceptance import ( + AcceptanceProduct, + build_manifest_acceptance_scope, + sdk_requirement, + validate_product_view, +) +from qdl.query import FeedType +from qdl.runtime.execution_l2 import execution_l2_materialization_plan +from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_deployment import StableAcquisitionPlan +from qdl_sdk.errors import DataLayerError +from scripts.phase103_consumer_receipt_acceptance import ( + _client, + _identity, + compact_feed_status, +) + + +DEFAULT_CATALOG = ROOT / "config/v2/stable-source-bindings.yaml" +DEFAULT_ACQUISITION = ROOT / "config/v2/stable-acquisition-bindings.yaml" +DEFAULT_EXECUTION_DEMAND = ROOT / "config/v2/stable-crypto-demand.yaml" +DEFAULT_TRADING_MANIFEST = ROOT / "consumers/stable/trading-system-paper.yaml" +EXPECTED_CONSUMER_ID = "trading-system.paper.stable" + + +def execution_book_products( + *, + catalog: StableSourceCatalog, + acquisition: StableAcquisitionPlan, + execution_demand: Path, + trading_manifest: Path, +) -> tuple[AcceptanceProduct, ...]: + """Join the Trading System manifest to all derived execution L2 sources.""" + + plan = execution_l2_materialization_plan( + demand_path=execution_demand, + catalog=catalog, + acquisition=acquisition, + ) + scope = build_manifest_acceptance_scope( + (trading_manifest,), + catalog=catalog, + acquisition=acquisition, + expected_consumer_ids=frozenset({EXPECTED_CONSUMER_ID}), + schema="qdl.phase105.consumer-acceptance-scope.v1", + requirement_filter=lambda item: item.feed is FeedType.BOOK_SNAPSHOT, + ) + source_by_binding = {item.binding_id: item.source_id for item in catalog.bindings} + products = tuple( + item + for item in scope.products + if item.binding_id is not None + and source_by_binding.get(item.binding_id) in plan.source_ids + ) + actual_ids = { + source_by_binding[item.binding_id] + for item in products + if item.binding_id is not None + } + if actual_ids != set(plan.source_ids) or len(products) != len(plan.source_ids): + raise ValueError("Trading System execution L2 matrix differs from the declared demand") + return tuple(sorted(products, key=lambda item: (item.venue, item.native_symbol))) + + +def compact_book_snapshot(view: object) -> dict[str, object]: + """Keep readiness evidence, never book levels or price/quantity payloads.""" + + payload = getattr(view, "payload", None) + source = getattr(view, "source", None) + quality = getattr(view, "quality", None) + fields = { + "source_id": getattr(source, "source_id", None), + "book_generation": getattr(payload, "book_generation", None), + "sequence_verified": getattr(payload, "sequence_verified", None), + "native_sequence": getattr(payload, "native_sequence", None), + "depth": getattr(payload, "depth", None), + "revision": getattr(view, "revision", None), + "watermark_offset": getattr(view, "watermark_offset", None), + "received_at_ns": getattr(view, "received_at_ns", None), + "event_age_ms": getattr(quality, "freshness_ms", None), + "gap_open": getattr(quality, "gap_open", None), + "complete": getattr(quality, "complete", None), + "execution_eligible": getattr(quality, "execution_eligible", None), + } + if ( + not isinstance(fields["source_id"], str) + or not fields["source_id"] + or not isinstance(fields["book_generation"], int) + or fields["book_generation"] < 0 + or not isinstance(fields["sequence_verified"], bool) + or not isinstance(fields["native_sequence"], str) + or not fields["native_sequence"] + or not isinstance(fields["depth"], int) + or fields["depth"] < 1 + or not isinstance(fields["revision"], int) + or fields["revision"] < 0 + or not isinstance(fields["watermark_offset"], int) + or fields["watermark_offset"] < 0 + or not isinstance(fields["received_at_ns"], int) + or fields["received_at_ns"] < 1 + or not isinstance(fields["event_age_ms"], int) + or fields["event_age_ms"] < 0 + or not isinstance(fields["gap_open"], bool) + or not isinstance(fields["complete"], bool) + or not isinstance(fields["execution_eligible"], bool) + ): + raise ValueError("execution L2 snapshot evidence has invalid typed fields") + return {**fields, "payload_recorded": False} + + +def ready_book_row(row: Mapping[str, object]) -> bool: + """Return true only for a fully verified execution-grade compact row.""" + + status = row.get("typed_status") + snapshot = row.get("snapshot") + if not isinstance(status, Mapping) or not isinstance(snapshot, Mapping): + return False + quality = status.get("quality") + return bool( + isinstance(quality, Mapping) + and quality.get("state") == "LIVE" + and quality.get("complete") is True + and quality.get("gap_open") is False + and quality.get("execution_eligible") is True + and snapshot.get("sequence_verified") is True + and isinstance(snapshot.get("book_generation"), int) + and int(snapshot["book_generation"]) >= 1 + and isinstance(snapshot.get("depth"), int) + and int(snapshot["depth"]) >= 100 + and snapshot.get("complete") is True + and snapshot.get("gap_open") is False + and snapshot.get("execution_eligible") is True + ) + + +def replica_parity(primary: Mapping[str, object], secondary: Mapping[str, object]) -> bool: + """Compare invariant identity/quality fields; native sequence may advance.""" + + for field in ( + "instrument_uid", + "venue", + "market", + "native_symbol", + "feed", + "source_policy_id", + "source_id", + "depth", + ): + if primary.get(field) != secondary.get(field): + return False + return ready_book_row(primary) and ready_book_row(secondary) + + +async def _read_one( + product: AcceptanceProduct, + *, + label: str, + base_url: str, + grpc_target: str, + identity, + state_dir: Path, + timeout_seconds: float, +) -> dict[str, object]: + requirement = sdk_requirement(product) + client = _client( + identity, + base_url=base_url, + grpc_target=grpc_target, + cursor_path=state_dir / f"{label}-{product.instrument_uid}.cursor", + timeout_seconds=timeout_seconds, + ) + source_id = "" + result: dict[str, object] = { + "typed_status": None, + "status_error": None, + "snapshot": None, + "snapshot_error": None, + } + try: + try: + status = await client.feed_status(requirement) + status_evidence = compact_feed_status(status) + except DataLayerError as error: + result["status_error"] = {"code": error.code, "detail": error.detail} + return { + "replica": label, + "instrument_uid": product.instrument_uid, + "instrument_id": product.instrument_id, + "venue": product.venue, + "market": product.market, + "native_symbol": product.native_symbol, + "feed": product.feed.value, + "source_policy_id": product.source_policy_id, + "source_id": source_id, + **result, + "payload_recorded": False, + } + result["typed_status"] = status_evidence + try: + response = await client.snapshot(requirement) + view = response.data + validate_product_view(product, view) + snapshot = compact_book_snapshot(view) + source_id = str(snapshot["source_id"]) + result["snapshot"] = snapshot + except DataLayerError as error: + result["snapshot_error"] = {"code": error.code, "detail": error.detail} + except ValueError as error: + result["snapshot_error"] = {"code": "INVALID_VIEW", "detail": str(error)} + finally: + await client.close() + return { + "replica": label, + "instrument_uid": product.instrument_uid, + "instrument_id": product.instrument_id, + "venue": product.venue, + "market": product.market, + "native_symbol": product.native_symbol, + "feed": product.feed.value, + "source_policy_id": product.source_policy_id, + "source_id": source_id, + **result, + "payload_recorded": False, + } + + +async def run(args: argparse.Namespace) -> dict[str, object]: + catalog = StableSourceCatalog.load(args.catalog) + acquisition = StableAcquisitionPlan.load(args.acquisition, catalog=catalog) + products = execution_book_products( + catalog=catalog, + acquisition=acquisition, + execution_demand=args.execution_demand, + trading_manifest=args.trading_manifest, + ) + identity = _identity( + product=products[0], + certificate_file=str(args.tls_certificate_file), + private_key_file=str(args.tls_private_key_file), + jwt_private_key_file=str(args.jwt_private_key_file), + jwt_key_id=args.jwt_key_id, + tls_ca_file=str(args.tls_ca_file), + issuer=args.issuer, + audience=args.audience, + ) + started = time.monotonic() + with tempfile.TemporaryDirectory(prefix="qdl-execution-l2-status-") as raw: + state_dir = Path(raw) + rows = [] + for product in products: + primary, secondary = await asyncio.gather( + _read_one( + product, + label="primary", + base_url=args.primary_url, + grpc_target=args.grpc_target, + identity=identity, + state_dir=state_dir, + timeout_seconds=args.timeout_seconds, + ), + _read_one( + product, + label="secondary", + base_url=args.secondary_url, + grpc_target=args.grpc_target, + identity=identity, + state_dir=state_dir, + timeout_seconds=args.timeout_seconds, + ), + ) + rows.append({ + "instrument_uid": product.instrument_uid, + "instrument_id": product.instrument_id, + "venue": product.venue, + "market": product.market, + "native_symbol": product.native_symbol, + "source_policy_id": product.source_policy_id, + "primary": primary, + "secondary": secondary, + "replica_parity": replica_parity(primary, secondary), + }) + ready = all( + row["replica_parity"] + and ready_book_row(row["primary"]) + and ready_book_row(row["secondary"]) + for row in rows + ) + return { + "schema": "qdl.phase105.execution-l2-status-matrix.v1", + "status": "PASS" if ready else "FAIL", + "consumer_id": EXPECTED_CONSUMER_ID, + "book_count": len(rows), + "replica_count": 2, + "rows": rows, + "elapsed_seconds": round(time.monotonic() - started, 3), + "provider_connections": 0, + "order_actions": 0, + "cursor_directory_removed": True, + "payload_recorded": False, + } + + +def parser() -> argparse.ArgumentParser: + value = argparse.ArgumentParser(description=__doc__) + value.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) + value.add_argument("--acquisition", type=Path, default=DEFAULT_ACQUISITION) + value.add_argument("--execution-demand", type=Path, default=DEFAULT_EXECUTION_DEMAND) + value.add_argument("--trading-manifest", type=Path, default=DEFAULT_TRADING_MANIFEST) + value.add_argument("--primary-url", required=True) + value.add_argument("--secondary-url", required=True) + value.add_argument("--grpc-target", required=True) + value.add_argument("--tls-ca-file", type=Path, required=True) + value.add_argument("--tls-certificate-file", type=Path, required=True) + value.add_argument("--tls-private-key-file", type=Path, required=True) + value.add_argument("--jwt-private-key-file", type=Path, required=True) + value.add_argument("--jwt-key-id", required=True) + value.add_argument("--issuer", default="https://identity.qdl.stable.internal") + value.add_argument("--audience", default="qdl-v2-stable") + value.add_argument("--timeout-seconds", type=float, default=15.0) + return value + + +def main() -> int: + args = parser().parse_args() + if not 5.0 <= args.timeout_seconds <= 60.0: + raise SystemExit("--timeout-seconds must be between 5 and 60") + result = asyncio.run(run(args)) + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return 0 if result["status"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/phase105_prepare_c2_identity_recovery.py b/scripts/phase105_prepare_c2_identity_recovery.py new file mode 100755 index 00000000..8fbaf517 --- /dev/null +++ b/scripts/phase105_prepare_c2_identity_recovery.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Prepare the public-only additive C2 external-identity recovery overlay.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from qdl.certification.phase105_handoff import ( + ALL_KEY_SUBJECTS, + RECOVERY_ALL_KEY_SUBJECTS, + RECOVERY_IDENTITY_SPECS, + handoff_packet, + load_dotenv, + prepare_c2_identity_recovery_environment, + public_handoff_overlay, + render_dotenv, + sha256_bytes, + validate_frozen_v1_provenance, +) + + +CONFIRM = "PREPARE_QDL_PHASE105C_IDENTITY_RECOVERY" + + +def _public_keyring(environment: dict[str, str]) -> dict[str, str]: + try: + value = json.loads(environment["QDL_STABLE_JWT_KEYS_JSON"]) + except (KeyError, json.JSONDecodeError) as error: + raise ValueError("Phase 10.5-C recovery base JWT keyring is invalid") from error + if not isinstance(value, dict) or set(value) != set(ALL_KEY_SUBJECTS): + raise ValueError("Phase 10.5-C recovery base JWT keyring is not the retained V1 set") + return value + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-env", type=Path, required=True) + parser.add_argument("--extension-dir", type=Path, required=True) + parser.add_argument("--reader-image", required=True) + parser.add_argument("--v1-provenance", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--apply", action="store_true") + parser.add_argument("--confirm") + args = parser.parse_args(argv) + if args.apply and args.confirm != CONFIRM: + raise SystemExit(f"--apply requires --confirm {CONFIRM}") + if args.output_dir.exists(): + raise SystemExit("Phase 10.5-C recovery output directory must not already exist") + + base = load_dotenv(args.base_env) + prior_keys = _public_keyring(base) + environment = prepare_c2_identity_recovery_environment( + base, + extension_dir=args.extension_dir, + python_image=args.reader_image, + ) + try: + v1_provenance_raw = json.loads(args.v1_provenance.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise SystemExit("Phase 10.5-C recovery V1 provenance cannot be read") from error + validate_frozen_v1_provenance(v1_provenance_raw) + packet = handoff_packet( + environment=environment, + extension_dir=args.extension_dir, + v1_attestation=v1_provenance_raw, + approved_key_subjects=RECOVERY_ALL_KEY_SUBJECTS, + ) + packet.update({ + "schema": "qdl.phase105c.identity-recovery.v1", + "status": "PREPARED", + "reader_image": args.reader_image, + "retained_key_ids": sorted(prior_keys), + "recovery_key_ids": sorted(RECOVERY_IDENTITY_SPECS), + }) + packet["packet_sha256"] = sha256_bytes( + json.dumps(packet, sort_keys=True, separators=(",", ":")).encode() + ) + overlay = public_handoff_overlay(environment) + preview = { + "schema": packet["schema"], + "status": "DRY_RUN" if not args.apply else "PREPARED", + "packet_sha256": packet["packet_sha256"], + "recreated_services": packet["recreated_services"], + "retained_key_ids": packet["retained_key_ids"], + "recovery_key_ids": packet["recovery_key_ids"], + "public_overlay_sha256": sha256_bytes(render_dotenv(overlay).encode()), + "secret_values_recorded": False, + } + if not args.apply: + print(json.dumps(preview, sort_keys=True)) + return 0 + + os.umask(0o077) + args.output_dir.mkdir(mode=0o700, parents=True) + env_path = args.output_dir / "identity-recovery-public.env" + packet_path = args.output_dir / "identity-recovery-packet.json" + env_path.write_text(render_dotenv(overlay), encoding="utf-8") + packet_path.write_text(json.dumps(packet, sort_keys=True, indent=2) + "\n", encoding="utf-8") + env_path.chmod(0o600) + packet_path.chmod(0o600) + print(json.dumps(preview | {"output_dir": str(args.output_dir)}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/phase105_prepare_external_consumer_extension.sh b/scripts/phase105_prepare_external_consumer_extension.sh index f2817348..7234179e 100755 --- a/scripts/phase105_prepare_external_consumer_extension.sh +++ b/scripts/phase105_prepare_external_consumer_extension.sh @@ -1,20 +1,25 @@ #!/usr/bin/env bash set -euo pipefail -# Generate only the new external paper-consumer material needed by Phase 10.5-B. -# It never receives the existing CA private key and must not rotate the Kafka or -# server TLS mesh. Query/stream may later trust client-ca-bundle.crt while they -# continue serving their existing certificate. +# Generate only additive external paper-consumer material. It never receives an +# existing CA private key and must not rotate the Kafka or server TLS mesh. +# Supplying QDL_PHASE105_EXISTING_CLIENT_BUNDLE preserves every active client +# CA while appending exactly one newly generated external CA. OUTPUT_DIR="${1:?usage: phase105_prepare_external_consumer_extension.sh OUTPUT_DIR SERVER_CA_FILE}" SERVER_CA_FILE="${2:?usage: phase105_prepare_external_consumer_extension.sh OUTPUT_DIR SERVER_CA_FILE}" CERT_DAYS="${QDL_PHASE105_EXTERNAL_CERT_DAYS:-90}" -REQUESTED_ROLES="${QDL_PHASE105_EXTERNAL_ROLES:-monitoring,alpha-okx,reference-l2}" +REQUESTED_ROLES="${QDL_PHASE105_EXTERNAL_ROLES:-monitoring,trading-system,alpha-binance,alpha-okx}" +EXISTING_CLIENT_BUNDLE="${QDL_PHASE105_EXISTING_CLIENT_BUNDLE:-${SERVER_CA_FILE}}" if [[ ! -f "${SERVER_CA_FILE}" ]]; then printf 'server CA file is unavailable: %s\n' "${SERVER_CA_FILE}" >&2 exit 64 fi +if [[ ! -f "${EXISTING_CLIENT_BUNDLE}" ]]; then + printf 'existing client trust bundle is unavailable: %s\n' "${EXISTING_CLIENT_BUNDLE}" >&2 + exit 64 +fi if [[ -e "${OUTPUT_DIR}" ]] && find "${OUTPUT_DIR}" -mindepth 1 -print -quit | grep -q .; then printf 'output directory must be empty: %s\n' "${OUTPUT_DIR}" >&2 exit 64 @@ -79,6 +84,8 @@ if [[ "${#roles[@]}" -eq 0 ]]; then fi declare -A subjects=( [monitoring]='spiffe://qdl/paper/monitoring-multivenue-stable' + [trading-system]='spiffe://qdl/paper/trading-system-stable' + [alpha-binance]='spiffe://qdl/paper/alpha-binance-stable' [alpha-okx]='spiffe://qdl/paper/alpha-okx-stable' [reference-l2]='spiffe://qdl/paper/reference-l2-stable' ) @@ -93,9 +100,9 @@ for role in "${roles[@]}"; do issue_jwt_key "${role}" done -# The first PEM is the active CA trusted by existing paper clients. The second -# is additive and signs only the newly introduced external consumers. -cat "${SERVER_CA_FILE}" "${EXTERNAL_CA_CERT}" >"${OUTPUT_DIR}/client-ca-bundle.crt" +# The first PEM set remains the exact active trust bundle. The final PEM is +# additive and signs only newly introduced external consumers. +cat "${EXISTING_CLIENT_BUNDLE}" "${EXTERNAL_CA_CERT}" >"${OUTPUT_DIR}/client-ca-bundle.crt" chmod 0444 "${OUTPUT_DIR}/client-ca-bundle.crt" "${EXTERNAL_CA_CERT}" rm -f "${EXTERNAL_CA_KEY}" "${OUTPUT_DIR}/external-client-ca.srl" diff --git a/scripts/phase115_render_consumer_route_binding.py b/scripts/phase115_render_consumer_route_binding.py index be0c4b14..ac38e508 100644 --- a/scripts/phase115_render_consumer_route_binding.py +++ b/scripts/phase115_render_consumer_route_binding.py @@ -14,6 +14,8 @@ from pathlib import Path from typing import Any, Mapping +import yaml + from qdl.consumer.universal_release import ( ConsumerRouteBinding, UniversalReleaseManifest, @@ -49,10 +51,36 @@ def _manifest_from_artifact(value: Mapping[str, Any]) -> UniversalReleaseManifes return manifest +def _consumer_manifest_revision(path: Path, *, consumer_id: str) -> int: + try: + value = yaml.safe_load(path.read_text(encoding="utf-8")) + except FileNotFoundError as error: + raise ValueError(f"consumer manifest is missing: {path}") from error + if not isinstance(value, Mapping): + raise ValueError("consumer manifest must be a mapping") + metadata = value.get("metadata") + if not isinstance(metadata, Mapping): + raise ValueError("consumer manifest metadata is missing") + if str(metadata.get("id", "")).strip() != consumer_id: + raise ValueError("consumer manifest id differs from requested consumer") + revision = metadata.get("revision") + if isinstance(revision, bool) or not isinstance(revision, int) or revision < 1: + raise ValueError("consumer manifest revision must be a positive integer") + return revision + + def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--release-artifact", type=Path, required=True) parser.add_argument("--consumer-id", required=True) + parser.add_argument( + "--consumer-manifest", + type=Path, + help=( + "Optional canonical consumer manifest. When supplied, render a " + "generation-bound v2 binding that seals metadata.revision." + ), + ) parser.add_argument("--output", type=Path, required=True) parser.add_argument( "--independent-v1-venue", @@ -64,10 +92,17 @@ def main(argv: list[str] | None = None) -> int: if args.release_artifact.resolve() == args.output.resolve(): raise ValueError("release artifact and binding output must be different files") manifest = _manifest_from_artifact(_read_mapping(args.release_artifact)) + consumer_id = str(args.consumer_id) + consumer_manifest_revision = ( + _consumer_manifest_revision(args.consumer_manifest, consumer_id=consumer_id) + if args.consumer_manifest is not None + else None + ) binding = ConsumerRouteBinding.from_manifest( manifest, - consumer_id=str(args.consumer_id), + consumer_id=consumer_id, independent_v1_venues=tuple(str(item) for item in args.independent_v1_venue), + consumer_manifest_revision=consumer_manifest_revision, ) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text( @@ -80,6 +115,7 @@ def main(argv: list[str] | None = None) -> int: "release_revision": binding.release_revision, "universal_manifest_sha256": binding.universal_manifest_sha256, "binding_sha256": binding.binding_sha256, + "consumer_manifest_revision": binding.consumer_manifest_revision, "product_count": len(binding.products), "runtime_mutations": 0, "order_actions": 0, diff --git a/scripts/phase12_materialize_bound_bar_edge.py b/scripts/phase12_materialize_bound_bar_edge.py index 82c91269..4eba9e56 100644 --- a/scripts/phase12_materialize_bound_bar_edge.py +++ b/scripts/phase12_materialize_bound_bar_edge.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 -"""Materialize the exact final-BAR runtime projection for one sealed consumer. +"""Materialize one bounded final-BAR projection from sealed V2 consumers. The stable catalog intentionally carries more venue/interval capability than a -single consumer needs. This control-plane tool derives a minimal catalog and -acquisition plan from a sealed V2 consumer-route binding so the shared BAR -edge cannot acquire an undeclared interval merely because it exists in the -image. It never contacts a provider or changes a running role. +declared consumer set needs. This control-plane tool derives a minimal catalog +and acquisition plan from sealed V2 consumer-route bindings plus, when +supplied, the exact route set already materialized for a retained active +consumer. The shared BAR edge cannot acquire an undeclared interval merely +because it exists in the image, and a new consumer union cannot silently remove +an active baseline route. It never contacts a provider or changes a running +role. """ from __future__ import annotations @@ -66,6 +69,21 @@ def _load_binding(path: Path) -> ConsumerRouteBinding: return ConsumerRouteBinding.from_canonical_mapping(value) +def _load_retained_projection(path: Path) -> tuple[dict[str, Any], str]: + """Load an active projection solely as a retained-route authority.""" + + try: + raw = path.read_bytes() + value = json.loads(raw) + except FileNotFoundError as error: + raise ValueError(f"retained BAR projection is missing: {path}") from error + except json.JSONDecodeError as error: + raise ValueError(f"retained BAR projection is not JSON: {path}") from error + if not isinstance(value, dict): + raise ValueError("retained BAR projection must be a JSON object") + return value, _sha256_bytes(raw) + + def _sha256_bytes(value: bytes) -> str: return hashlib.sha256(value).hexdigest() @@ -80,7 +98,6 @@ def _bar_key_from_product(product: object) -> tuple[str, ...]: if ( getattr(product, "route") != "V2_PRIMARY" or not getattr(product, "require_final_bars") - or not getattr(product, "require_live") or getattr(product, "provider_plane") != "REALTIME" or not getattr(product, "interval") ): @@ -111,6 +128,55 @@ def _bar_key_from_source(source: object) -> tuple[str, ...]: ) +def _retained_projection_consumer_ids( + projection: Mapping[str, Any] | None, +) -> tuple[str, ...]: + """Read the consumer identity from a legacy or union projection receipt.""" + + if projection is None: + return () + singular = projection.get("consumer_id") + plural = projection.get("consumer_ids") + if isinstance(singular, str) and singular: + if plural is not None: + if ( + not isinstance(plural, list) + or any(not isinstance(item, str) or not item for item in plural) + or set(plural) != {singular} + ): + raise ValueError("retained BAR projection consumer identities are invalid") + return (singular,) + if ( + isinstance(plural, list) + and plural + and all(isinstance(item, str) and item for item in plural) + ): + return tuple(sorted(set(plural))) + raise ValueError("retained BAR projection consumer identity is invalid") + + +def _retained_bar_binding_ids(projection: Mapping[str, Any] | None) -> tuple[str, ...]: + """Return only a verified active final-BAR baseline route set.""" + + if projection is None: + return () + if projection.get("schema") != "qdl.phase12.bound-bar-edge-projection.v1": + raise ValueError("retained BAR projection schema is invalid") + if projection.get("status") != "MATERIALIZED": + raise ValueError("retained BAR projection is not materialized") + _retained_projection_consumer_ids(projection) + for field in ("catalog_sha256", "acquisition_sha256"): + value = projection.get(field) + if not isinstance(value, str) or len(value) != 64: + raise ValueError(f"retained BAR projection {field} is invalid") + raw_ids = projection.get("binding_ids") + if not isinstance(raw_ids, list) or not raw_ids or any( + not isinstance(item, str) or not item for item in raw_ids + ): + raise ValueError("retained BAR projection binding_ids are invalid") + return tuple(sorted(set(raw_ids))) + + def _validate_projection( *, catalog: Mapping[str, Any], @@ -173,18 +239,22 @@ def _project_acquisition_for_python_bar_edge( ) -def build_bound_bar_projection( +def build_bound_bar_projection_set( *, - binding: ConsumerRouteBinding, + bindings: tuple[ConsumerRouteBinding, ...], catalog_document: Mapping[str, Any], acquisition_document: Mapping[str, Any], + retained_projection: Mapping[str, Any] | None = None, + retained_projection_sha256: str | None = None, ) -> BoundBarProjection: - """Return the exact final-BAR catalog/acquisition subset for ``binding``. + """Return the exact final-BAR subset for a declared sealed consumer set. This is intentionally derived by canonical identity rather than source binding ID: a source catalog may change an implementation identifier, but it may never silently change venue, contract, symbol, policy or interval. """ + if not bindings: + raise ValueError("bound BAR projection requires at least one consumer binding") full_catalog = StableSourceCatalog.from_mapping(catalog_document) full_catalog_ids = {item.binding_id for item in full_catalog.bindings} @@ -201,13 +271,13 @@ def build_bound_bar_projection( requested = tuple( _bar_key_from_product(product) + for binding in bindings for product in binding.products if product.feed == "BAR" ) if not requested: - raise ValueError("consumer route binding has no BAR product") - if len(requested) != len(set(requested)): - raise ValueError("consumer route binding has duplicate BAR identities") + raise ValueError("consumer route bindings have no BAR product") + requested_unique = tuple(sorted(set(requested))) by_key: dict[tuple[str, ...], list[object]] = {} for source in full_catalog.bindings: @@ -216,7 +286,7 @@ def build_bound_bar_projection( by_key.setdefault(_bar_key_from_source(source), []).append(source) selected_ids: list[str] = [] - for key in requested: + for key in requested_unique: matches = by_key.get(key, []) if len(matches) != 1: raise ValueError( @@ -228,8 +298,25 @@ def build_bound_bar_projection( raise ValueError("sealed BAR route resolved to a non-final catalog binding") selected_ids.append(source.binding_id) - selected = frozenset(selected_ids) - if len(selected) != len(requested) or not selected <= full_catalog_ids: + retained_consumer_ids = _retained_projection_consumer_ids(retained_projection) + retained_ids = _retained_bar_binding_ids(retained_projection) + for binding_id in retained_ids: + source = source_by_binding_id.get(binding_id) + if source is None: + raise ValueError( + "retained BAR route no longer resolves to a catalog binding: " + binding_id + ) + if source.feed.value != "BAR" or not source.require_final_bar: + raise ValueError( + "retained BAR route is not a final catalog BAR binding: " + binding_id + ) + + selected = frozenset((*selected_ids, *retained_ids)) + if ( + len(set(selected_ids)) != len(requested_unique) + or not selected + or not selected <= full_catalog_ids + ): raise ValueError("sealed BAR selection is invalid") source_raw_by_id = { str(item.get("binding_id")): item @@ -280,11 +367,13 @@ def build_bound_bar_projection( acquisition["bindings"] = selected_acquisition _validate_projection(catalog=catalog, acquisition=acquisition) - summary = { + consumer_ids = tuple(sorted({binding.consumer_id for binding in bindings})) + binding_sha256s = tuple(sorted({binding.binding_sha256 for binding in bindings})) + summary: dict[str, Any] = { "schema": "qdl.phase12.bound-bar-edge-projection.v1", "status": "MATERIALIZED", - "consumer_id": binding.consumer_id, - "binding_sha256": binding.binding_sha256, + "consumer_ids": list(consumer_ids), + "binding_sha256s": list(binding_sha256s), "bar_route_count": len(selected), "binding_ids": sorted(selected), "catalog_sha256": _sha256_bytes(_yaml_bytes(catalog)), @@ -293,9 +382,37 @@ def build_bound_bar_projection( "provider_requests": 0, "order_actions": 0, } + if retained_projection is not None: + retained_summary = { + "binding_ids": list(retained_ids), + "sha256": retained_projection_sha256, + } + if len(retained_consumer_ids) == 1: + retained_summary["consumer_id"] = retained_consumer_ids[0] + else: + retained_summary["consumer_ids"] = list(retained_consumer_ids) + summary["retained_projection"] = retained_summary + if len(bindings) == 1: + summary["consumer_id"] = bindings[0].consumer_id + summary["binding_sha256"] = bindings[0].binding_sha256 return BoundBarProjection(catalog=catalog, acquisition=acquisition, summary=summary) +def build_bound_bar_projection( + *, + binding: ConsumerRouteBinding, + catalog_document: Mapping[str, Any], + acquisition_document: Mapping[str, Any], +) -> BoundBarProjection: + """Backward-compatible single-binding form of the union compiler.""" + + return build_bound_bar_projection_set( + bindings=(binding,), + catalog_document=catalog_document, + acquisition_document=acquisition_document, + ) + + def _write_projection(*, output_dir: Path, projection: BoundBarProjection) -> None: if output_dir.exists(): raise ValueError(f"output directory already exists: {output_dir}") @@ -319,7 +436,18 @@ def _write_projection(*, output_dir: Path, projection: BoundBarProjection) -> No def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--consumer-binding", type=Path, required=True) + parser.add_argument( + "--consumer-binding", + type=Path, + required=True, + action="append", + help="sealed V2 consumer binding; repeat to project their exact BAR union", + ) + parser.add_argument( + "--retain-projection", + type=Path, + help="active materialized BAR projection whose exact baseline routes must remain", + ) parser.add_argument( "--catalog", type=Path, default=ROOT / "config/v2/stable-source-bindings.yaml" ) @@ -333,14 +461,30 @@ def main(argv: list[str] | None = None) -> int: if args.apply and args.confirm != CONFIRMATION: raise SystemExit(f"--apply requires --confirm {CONFIRMATION}") if args.output_dir.resolve() in { - args.consumer_binding.resolve(), args.catalog.resolve(), args.acquisition.resolve() + *[path.resolve() for path in args.consumer_binding], + *( + (args.retain_projection.resolve(),) + if args.retain_projection is not None + else () + ), + args.catalog.resolve(), + args.acquisition.resolve(), }: raise SystemExit("output directory must differ from every input file") - projection = build_bound_bar_projection( - binding=_load_binding(args.consumer_binding), + retained_projection = None + retained_projection_sha256 = None + if args.retain_projection is not None: + retained_projection, retained_projection_sha256 = _load_retained_projection( + args.retain_projection + ) + + projection = build_bound_bar_projection_set( + bindings=tuple(_load_binding(path) for path in args.consumer_binding), catalog_document=_load_mapping(args.catalog, label="stable source catalog"), acquisition_document=_load_mapping(args.acquisition, label="stable acquisition"), + retained_projection=retained_projection, + retained_projection_sha256=retained_projection_sha256, ) if args.apply: _write_projection(output_dir=args.output_dir, projection=projection) diff --git a/scripts/phase533_materialize_alpha_runtime_entitlements.py b/scripts/phase533_materialize_alpha_runtime_entitlements.py new file mode 100644 index 00000000..1d321a64 --- /dev/null +++ b/scripts/phase533_materialize_alpha_runtime_entitlements.py @@ -0,0 +1,493 @@ +"""Render bounded five-liquid V2 alpha entitlements from canonical demand. + +The output is a declaration only. It never opens a provider connection, +creates acquisition demand, or changes a runtime bundle. Shared Rust ingest, +bar-edge and projector roles stay the only producers for these routes. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +import hashlib +import json +from pathlib import Path +from typing import Any, Mapping + +import yaml + +from qdl.consumer import ConsumerManifestLoader, requirement_key + + +ROOT = Path(__file__).resolve().parents[1] +_TARGETS = { + "alpha.binance.paper.stable": { + "filename": "alpha-binance-paper.yaml", + "venue": "BINANCE", + "market": "USDM", + "product_type": "PERPETUAL", + }, + "alpha.okx.paper.stable": { + "filename": "alpha-okx-paper.yaml", + "venue": "OKX", + "market": "SWAP", + "product_type": "PERPETUAL", + }, +} +_REALTIME_FEEDS = frozenset({ + "BAR", "TRADE", "QUOTE", "BOOK_SNAPSHOT", "BOOK_DELTA", +}) +_REFERENCE_FEEDS = frozenset({ + "FUNDING_RATE", "OPEN_INTEREST", "LONG_SHORT_RATIO", "TAKER_FLOW", + "MARK_INDEX_PRICE", "CONTRACT_METADATA", "BASIS", +}) +_FEED_ORDER = { + "TRADE": 0, + "QUOTE": 1, + "BAR": 2, + "BOOK_SNAPSHOT": 3, + "BOOK_DELTA": 4, +} +_INTERVAL_ORDER = { + interval: index + for index, interval in enumerate( + ("1m", "3m", "5m", "15m", "30m", "1h", "2h", "4h", "6h", "8h", "12h", "1d", "2d", "3d", "1w") + ) +} +_BLOCKED_ROUTE_REASONS = { + "BAR": "V1_FINAL_BAR_EQUIVALENCE_UNPROVEN", + "QUOTE": "V1_QUOTE_EQUIVALENCE_UNPROVEN", + "BOOK_SNAPSHOT": "V1_L2_EQUIVALENCE_UNPROVEN", + "BOOK_DELTA": "V1_L2_EQUIVALENCE_UNPROVEN", +} +_REFERENCE_ROUTE = { + "route": "V2_PRIMARY", + "fallback": "BLOCKED", + "reason": "V1_REFERENCE_EQUIVALENCE_UNPROVEN", +} + + +def _yaml_bytes(value: Mapping[str, Any]) -> bytes: + return yaml.safe_dump(dict(value), sort_keys=False).encode("utf-8") + + +def _load(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"{path} must be a YAML mapping") + return payload + + +def _write_if_changed(path: Path, payload: Mapping[str, Any]) -> bool: + value = _yaml_bytes(payload) + if path.read_bytes() == value: + return False + path.write_bytes(value) + return True + + +def _identity(row: Mapping[str, Any]) -> tuple[str, str, str, str, str, str | None, str]: + return ( + str(row["venue"]).upper(), + str(row["market"]).upper(), + str(row["product_type"]).upper(), + str(row["native_symbol"]).upper(), + str(row["feed"]).upper(), + str(row["interval"]).lower() if row.get("interval") is not None else None, + str(row["source_policy_id"]), + ) + + +def _catalog_indexes( + catalog: Mapping[str, Any], +) -> tuple[dict[str, Mapping[str, Any]], dict[tuple[str, str, str | None, str], Mapping[str, Any]]]: + instruments = { + str(item["instrument_uid"]): item + for item in catalog.get("instruments", []) + if isinstance(item, Mapping) + } + bindings: dict[tuple[str, str, str | None, str], Mapping[str, Any]] = {} + for item in catalog.get("bindings", []): + if not isinstance(item, Mapping): + continue + source = item.get("source") + if not isinstance(source, Mapping): + raise ValueError("stable source binding has no source mapping") + key = ( + str(item["instrument_uid"]), + str(item["feed"]).upper(), + str(item["interval"]).lower() if item.get("interval") is not None else None, + str(source["source_policy_id"]), + ) + if key in bindings: + raise ValueError(f"stable source binding is duplicated: {key}") + bindings[key] = item + return instruments, bindings + + +def _target_uids( + manifest: Mapping[str, Any], + *, + target: Mapping[str, str], + instruments: Mapping[str, Mapping[str, Any]], +) -> tuple[str, ...]: + values: list[str] = [] + for item in manifest["spec"]["requirements"]: + if ( + item.get("feed") != "TRADE" + or item.get("source_policy_id") != "crypto_primary_v2" + ): + continue + uid = str(item["instrument_uid"]) + instrument = instruments.get(uid) + if instrument is None: + raise ValueError(f"alpha manifest references an unknown instrument: {uid}") + if all( + str(instrument.get(field, "")).upper() == target[field] + for field in ("venue", "market", "product_type") + ) and uid not in values: + values.append(uid) + if len(values) != 5: + raise ValueError("alpha runtime entitlement requires exactly five liquid native instruments") + # The source manifest order is not a semantic identity. Sorting makes the + # generated artifact stable after its first revision bump. + return tuple(sorted(values)) + + +def _demand_rows( + demand: Mapping[str, Any], + *, + instrument: Mapping[str, Any], +) -> list[Mapping[str, Any]]: + expected = ( + str(instrument["venue"]).upper(), + str(instrument["market"]).upper(), + str(instrument["product_type"]).upper(), + str(instrument["native_symbol"]).upper(), + ) + values: list[Mapping[str, Any]] = [] + for consumer in demand.get("consumers", []): + if not isinstance(consumer, Mapping): + continue + for row in consumer.get("requirements", []): + if not isinstance(row, Mapping) or str(row.get("feed", "")).upper() not in _REALTIME_FEEDS: + continue + identity = ( + str(row.get("venue", "")).upper(), + str(row.get("market", "")).upper(), + str(row.get("product_type", "")).upper(), + str(row.get("native_symbol", "")).upper(), + ) + if identity == expected: + values.append(row) + return values + + +def _manifest_requirement( + row: Mapping[str, Any], + *, + instrument_uid: str, + binding: Mapping[str, Any], +) -> dict[str, Any]: + feed = str(row["feed"]).upper() + quality = binding.get("quality") + if not isinstance(quality, Mapping): + raise ValueError("stable source binding has no quality mapping") + freshness = row.get("max_freshness_ms", quality.get("stale_after_ms")) + if not isinstance(freshness, int) or isinstance(freshness, bool) or freshness <= 0: + raise ValueError("stable source binding freshness is invalid") + result: dict[str, Any] = { + "instrument_uid": instrument_uid, + "feed": feed, + "consumer_grade": "ALPHA", + "source_policy_id": str(row["source_policy_id"]), + "interval": row.get("interval"), + "warmup_limit": 10_000 if feed == "BAR" else 0, + "max_freshness_ms": freshness, + "require_full_coverage": True, + "require_final_bars": bool(quality.get("require_final_bar", False)) if feed == "BAR" else False, + "stale_policy": "BLOCK", + "gap_policy": "BLOCK", + "recovery": "SNAPSHOT_AND_REPLAY", + "bar_revision_policy": "EMIT_REVISIONS" if feed == "BAR" else "LATEST", + } + if feed == "TRADE": + result["event_recency_policy"] = "OBSERVE" + if feed in {"TRADE", "QUOTE", "BOOK_SNAPSHOT", "BOOK_DELTA"}: + result["max_session_liveness_ms"] = 45_000 + return result + + +def _realtime_templates( + *, + manifest: Mapping[str, Any], + demand: Mapping[str, Any], + instruments: Mapping[str, Mapping[str, Any]], + bindings: Mapping[tuple[str, str, str | None, str], Mapping[str, Any]], + target: Mapping[str, str], +) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for uid in _target_uids(manifest, target=target, instruments=instruments): + instrument = instruments[uid] + source_rows = _demand_rows(demand, instrument=instrument) + keys = {_identity(row) for row in source_rows} + if len(keys) != len(source_rows): + raise ValueError("stable crypto demand has duplicate alpha runtime identities") + expected = { + "TRADE": 1, + "QUOTE": 1, + "BOOK_SNAPSHOT": 1, + "BOOK_DELTA": 1, + "BAR": 14, + } + counts = { + feed: sum(str(row["feed"]).upper() == feed for row in source_rows) + for feed in expected + } + if counts != expected: + raise ValueError( + f"stable crypto demand is incomplete for {instrument['native_symbol']}: {counts}" + ) + for source_row in source_rows: + feed = str(source_row["feed"]).upper() + interval = ( + str(source_row["interval"]).lower() + if source_row.get("interval") is not None + else None + ) + policy_id = str(source_row["source_policy_id"]) + try: + binding = bindings[(uid, feed, interval, policy_id)] + except KeyError as error: + raise ValueError( + f"stable source catalog misses {uid}/{feed}/{interval}/{policy_id}" + ) from error + rows.append( + _manifest_requirement(source_row, instrument_uid=uid, binding=binding) + ) + rows.sort( + key=lambda item: ( + item["instrument_uid"], + _FEED_ORDER[item["feed"]], + _INTERVAL_ORDER.get(item.get("interval"), -1), + ) + ) + return rows + + +def _reference_templates( + reference_manifest: Mapping[str, Any], +) -> dict[str, list[dict[str, Any]]]: + result: dict[str, list[dict[str, Any]]] = {} + for item in reference_manifest["spec"]["requirements"]: + if ( + str(item.get("feed")) not in _REFERENCE_FEEDS + or str(item.get("source_policy_id")) != "crypto_liquid_v2" + ): + continue + result.setdefault(str(item["instrument_uid"]), []).append(deepcopy(item)) + for values in result.values(): + values.sort(key=lambda item: (str(item["feed"]), str(item.get("interval") or ""))) + return result + + +def _is_managed(item: Mapping[str, Any]) -> bool: + feed = str(item.get("feed", "")) + return ( + feed in _REALTIME_FEEDS + or (feed in _REFERENCE_FEEDS and str(item.get("source_policy_id")) == "crypto_liquid_v2") + ) + + +def _materialize_manifest( + manifest: Mapping[str, Any], + *, + realtime: list[dict[str, Any]], + reference_templates: Mapping[str, list[dict[str, Any]]], + target: Mapping[str, str], + instruments: Mapping[str, Mapping[str, Any]], +) -> tuple[dict[str, Any], bool]: + result = deepcopy(dict(manifest)) + existing = list(result["spec"]["requirements"]) + uids = _target_uids(result, target=target, instruments=instruments) + references: list[dict[str, Any]] = [] + for uid in uids: + try: + rows = reference_templates[uid] + except KeyError as error: + raise ValueError(f"reference capability has no template for {uid}") from error + for row in rows: + value = deepcopy(row) + value["consumer_grade"] = "ALPHA" + references.append(value) + base = [item for item in existing if not _is_managed(item)] + requirements = [*base, *realtime, *references] + result["spec"]["requirements"] = requirements + result["spec"]["quotas"]["max_warmup_rows"] = 10_000 + changed = ( + requirements != existing + or int(manifest["spec"]["quotas"]["max_warmup_rows"]) != 10_000 + ) + if changed: + result["metadata"]["revision"] = int(result["metadata"]["revision"]) + 1 + return result, changed + + +def _default_route( + requirement: Any, + *, + bindings: Mapping[tuple[str, str, str | None, str], Mapping[str, Any]], +) -> dict[str, Any]: + feed = requirement.feed.value + if feed in _REFERENCE_FEEDS: + return dict(_REFERENCE_ROUTE) + key = ( + requirement.instrument_uid, + feed, + requirement.interval, + requirement.source_policy_id, + ) + binding = bindings.get(key) + if binding is None: + raise ValueError(f"route source binding is missing: {key}") + if feed == "TRADE" and str(binding.get("v1_compatibility", "NONE")) != "NONE": + return {"route": "V2_PRIMARY", "fallback": "V1", "reason": None} + return { + "route": "V2_PRIMARY", + "fallback": "BLOCKED", + "reason": _BLOCKED_ROUTE_REASONS.get(feed, "V1_EQUIVALENCE_UNPROVEN"), + } + + +def _materialize_route( + route: Mapping[str, Any], + *, + manifests: Mapping[str, Mapping[str, Any]], + bindings: Mapping[tuple[str, str, str | None, str], Mapping[str, Any]], +) -> tuple[dict[str, Any], bool]: + result = deepcopy(dict(route)) + targets = {str(item["consumer_id"]): item for item in result["consumers"]} + changed = False + for consumer_id, payload in manifests.items(): + target = targets.get(consumer_id) + if target is None: + raise ValueError(f"stable release route has no consumer {consumer_id}") + manifest = ConsumerManifestLoader.from_mapping(payload) + current = { + str(item["requirement_key"]): item for item in target["products"] + } + products = [] + for requirement in manifest.requirements: + key = requirement_key(requirement) + existing = current.get(key) + products.append( + {"requirement_key": key, **(deepcopy(existing) if existing is not None else _default_route(requirement, bindings=bindings))} + if existing is None + else deepcopy(existing) + ) + if ( + int(target["manifest_revision"]) != manifest.manifest_revision + or str(target["manifest_sha256"]) != manifest.manifest_sha256 + or products != target["products"] + ): + changed = True + target["manifest_revision"] = manifest.manifest_revision + target["manifest_sha256"] = manifest.manifest_sha256 + target["products"] = products + if changed: + result["revision"] = int(result["revision"]) + 1 + return result, changed + + +def build_documents( + *, + catalog: Mapping[str, Any], + demand: Mapping[str, Any], + reference_manifest: Mapping[str, Any], + alpha_manifests: Mapping[str, Mapping[str, Any]], + release_route: Mapping[str, Any], + primary_route: Mapping[str, Any], +) -> tuple[dict[str, dict[str, Any]], dict[str, Any], dict[str, Any], dict[str, object]]: + instruments, bindings = _catalog_indexes(catalog) + reference_templates = _reference_templates(reference_manifest) + manifests: dict[str, dict[str, Any]] = {} + manifest_changed = False + for consumer_id, target in _TARGETS.items(): + realtime = _realtime_templates( + manifest=alpha_manifests[consumer_id], + demand=demand, + instruments=instruments, + bindings=bindings, + target=target, + ) + manifest, changed = _materialize_manifest( + alpha_manifests[consumer_id], + realtime=realtime, + reference_templates=reference_templates, + target=target, + instruments=instruments, + ) + manifests[consumer_id] = manifest + manifest_changed = manifest_changed or changed + route, route_changed = _materialize_route( + release_route, + manifests=manifests, + bindings=bindings, + ) + primary = deepcopy(dict(primary_route)) + if manifest_changed: + primary["revision"] = int(primary["revision"]) + 1 + summary = { + "schema": "qdl.phase533.alpha-runtime-entitlements.v1", + "status": "READY", + "manifest_changed": manifest_changed, + "release_route_changed": route_changed, + "alpha_requirement_counts": { + consumer_id: len(payload["spec"]["requirements"]) + for consumer_id, payload in manifests.items() + }, + "manifest_revisions": { + consumer_id: int(payload["metadata"]["revision"]) + for consumer_id, payload in manifests.items() + }, + "release_revision": int(route["revision"]), + "primary_route_revision": int(primary["revision"]), + } + return manifests, route, primary, summary + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--catalog", type=Path, default=ROOT / "config/v2/stable-source-bindings.yaml") + parser.add_argument("--demand", type=Path, default=ROOT / "config/v2/stable-crypto-demand.yaml") + parser.add_argument("--reference-manifest", type=Path, default=ROOT / "consumers/stable/reference-l2-stable.yaml") + parser.add_argument("--release-route", type=Path, default=ROOT / "config/v2/stable-v2-release-routing.yaml") + parser.add_argument("--primary-route", type=Path, default=ROOT / "config/v2/stable-primary-consumer-routing.yaml") + parser.add_argument("--apply", action="store_true") + args = parser.parse_args(argv) + paths = { + consumer_id: ROOT / "consumers/stable" / target["filename"] + for consumer_id, target in _TARGETS.items() + } + manifests, route, primary, summary = build_documents( + catalog=_load(args.catalog), + demand=_load(args.demand), + reference_manifest=_load(args.reference_manifest), + alpha_manifests={consumer_id: _load(path) for consumer_id, path in paths.items()}, + release_route=_load(args.release_route), + primary_route=_load(args.primary_route), + ) + changed_files: list[str] = [] + if args.apply: + for consumer_id, path in paths.items(): + if _write_if_changed(path, manifests[consumer_id]): + changed_files.append(str(path)) + if _write_if_changed(args.release_route, route): + changed_files.append(str(args.release_route)) + if _write_if_changed(args.primary_route, primary): + changed_files.append(str(args.primary_route)) + print(json.dumps({**summary, "changed_files": changed_files, "status": "APPLIED" if args.apply else "DRY_RUN"}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/phasec36_reference_l2_consumer_acceptance.py b/scripts/phasec36_reference_l2_consumer_acceptance.py index 71daf1de..71508f2f 100644 --- a/scripts/phasec36_reference_l2_consumer_acceptance.py +++ b/scripts/phasec36_reference_l2_consumer_acceptance.py @@ -121,7 +121,8 @@ async def _reference_batch_until_terminal( remaining_ms = int((deadline_monotonic - clock()) * 1_000) if retry_after_ms + 250 >= remaining_ms: raise AssertionError( - "native BASIS cooldown exceeds the bounded Reference/L2 acceptance deadline" + "native BASIS cooldown exceeds the bounded Reference/L2 acceptance deadline " + f"retry_after_ms={retry_after_ms} remaining_ms={remaining_ms}" ) await sleep(retry_after_ms / 1_000) deferred_ms += retry_after_ms diff --git a/scripts/prepare_alpha_reader_release.py b/scripts/prepare_alpha_reader_release.py new file mode 100644 index 00000000..20ee5700 --- /dev/null +++ b/scripts/prepare_alpha_reader_release.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +"""Seal a secret-free reader release from config-derived alpha bindings.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +from typing import Any, Mapping +import uuid + +from qdl.consumer.universal_release import ConsumerRouteBinding + + +CONFIRM = "PREPARE_QDL_ALPHA_READER_RELEASE" +_PRIVATE_ARTIFACT_MODE = 0o640 +# Route bindings contain hashes, identity and policy only. They are mounted +# directly into non-root alpha workloads, so the release copy must be readable +# by that workload without making any credential artifact readable. +_PUBLIC_BINDING_MODE = 0o444 +SCHEMA = "qdl.v2.alpha-reader-release.v2" +READER_SERVICES = ( + "query_v2_1", + "query_v2_2", + "stream_v2_active", + "stream_v2_passive", +) +_DIGEST = re.compile(r"[0-9a-f]{64}\Z") +_IMAGE_ID = re.compile(r"sha256:[0-9a-f]{64}\Z") +_SOURCE_REVISION = re.compile(r"[0-9a-f]{40}\Z") +_RELEASE_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{2,120}\Z") +_BINDING_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,180}\.binding\.json\Z") +_IMAGE_REFERENCE = re.compile(r"qdl-v2-python:[A-Za-z0-9][A-Za-z0-9._-]{0,127}\Z") + + +class ReleasePreparationError(ValueError): + """Raised when a release input cannot be sealed safely.""" + + +def _canonical_bytes(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def _digest(value: object) -> str: + return hashlib.sha256(_canonical_bytes(value)).hexdigest() + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _load_mapping(path: Path, label: str) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as error: + raise ReleasePreparationError(f"{label} is missing") from error + except json.JSONDecodeError as error: + raise ReleasePreparationError(f"{label} is not valid JSON") from error + if not isinstance(value, dict): + raise ReleasePreparationError(f"{label} must be a JSON object") + return value + + +def _require_digest(value: object, label: str) -> str: + result = str(value or "").strip().lower() + if not _DIGEST.fullmatch(result): + raise ReleasePreparationError(f"{label} must be a SHA-256 digest") + return result + + +def _validate_inventory(path: Path) -> tuple[dict[str, Any], str]: + inventory = _load_mapping(path, "inventory") + expected = { + "schema", "revision", "registry_path", "registry_sha256", "deployments", "inventory_sha256", + } + if set(inventory) != expected or inventory.get("schema") != "execution-alpha.data-requirements.v1": + raise ReleasePreparationError("inventory schema or fields are invalid") + reported = _require_digest(inventory.get("inventory_sha256"), "inventory_sha256") + unsigned = dict(inventory) + unsigned.pop("inventory_sha256") + if _digest(unsigned) != reported: + raise ReleasePreparationError("inventory checksum differs") + if not isinstance(inventory.get("deployments"), list) or not inventory["deployments"]: + raise ReleasePreparationError("inventory deployments are required") + return inventory, reported + + +def _validate_report(path: Path, *, inventory_sha256: str) -> dict[str, Any]: + report = _load_mapping(path, "compilation report") + expected = { + "schema", "contract_version", "inventory_sha256", "catalog_sha256", + "reference_manifest_sha256", "release_routing_sha256", "policy_sha256", + "deployments", "compilation_sha256", + } + if set(report) != expected or report.get("schema") != "qdl.v2.alpha-deployment-binding-compilation.v1": + raise ReleasePreparationError("compilation report schema or fields are invalid") + if _require_digest(report.get("inventory_sha256"), "report inventory_sha256") != inventory_sha256: + raise ReleasePreparationError("compilation report inventory checksum differs") + reported = _require_digest(report.get("compilation_sha256"), "compilation_sha256") + unsigned = dict(report) + unsigned.pop("compilation_sha256") + if _digest(unsigned) != reported: + raise ReleasePreparationError("compilation report checksum differs") + deployments = report.get("deployments") + if not isinstance(deployments, list): + raise ReleasePreparationError("compilation report deployments are invalid") + return report + + +def _canonical_binding(value: Mapping[str, Any], label: str) -> dict[str, Any]: + try: + canonical = ConsumerRouteBinding.from_canonical_mapping(value).canonical_mapping() + except (KeyError, TypeError, ValueError) as error: + raise ReleasePreparationError(f"{label} is invalid") from error + if dict(value) != canonical: + raise ReleasePreparationError(f"{label} is not canonical") + return canonical + + +def _binding_digests(report: Mapping[str, Any]) -> set[str]: + result: set[str] = set() + for deployment in report["deployments"]: + if not isinstance(deployment, Mapping): + raise ReleasePreparationError("compilation deployment is invalid") + if deployment.get("status") != "ADMITTED": + continue + binding = deployment.get("binding") + if not isinstance(binding, Mapping): + raise ReleasePreparationError("admitted deployment binding is missing") + _canonical_binding(binding, "admitted deployment binding") + digest = _require_digest(binding.get("binding_sha256"), "binding_sha256") + if digest in result: + raise ReleasePreparationError("admitted deployment binding is duplicated") + result.add(digest) + if not result: + raise ReleasePreparationError("compilation report has no admitted bindings") + return result + + +def _validate_binding_files(directory: Path, expected_digests: set[str]) -> tuple[Path, ...]: + if not directory.is_dir(): + raise ReleasePreparationError("binding directory is missing") + files = tuple(sorted(directory.glob("*.binding.json"))) + if len(files) != len(expected_digests): + raise ReleasePreparationError("binding file count differs from compilation report") + observed: set[str] = set() + for path in files: + if not _BINDING_NAME.fullmatch(path.name): + raise ReleasePreparationError("binding filename is unsafe") + binding = _load_mapping(path, f"binding {path.name}") + _canonical_binding(binding, "binding file") + digest = _require_digest(binding.get("binding_sha256"), "binding file binding_sha256") + if digest in observed: + raise ReleasePreparationError("binding file digest is duplicated") + observed.add(digest) + if observed != expected_digests: + raise ReleasePreparationError("binding files differ from compilation report") + return files + + +def _validate_output(path: Path, *, inventory: Path, bindings: Path, report: Path) -> Path: + output = path.resolve() + if not _RELEASE_NAME.fullmatch(output.name): + raise ReleasePreparationError("output release directory name is unsafe") + if output.exists(): + raise FileExistsError("output release directory already exists") + if not output.parent.is_dir(): + raise FileNotFoundError("output release parent must already exist") + for source in (inventory.resolve(), bindings.resolve(), report.resolve()): + if output == source or source in output.parents: + raise ReleasePreparationError("output release directory must not contain an input artifact") + return output + + +def _image_selector(image_reference: str, image_id: str) -> str: + """Pin a readable local tag to the immutable image ID Compose must run.""" + + return f"{image_reference}@{image_id}" + + +def _render_override(images_by_service: Mapping[str, Mapping[str, str]]) -> bytes: + return ( + "services:\n" + + "".join( + " " + + service + + ":\n image: " + + _image_selector( + images_by_service[service]["image_reference"], + images_by_service[service]["image_id"], + ) + + "\n" + for service in READER_SERVICES + ) + ).encode("utf-8") + + +def _validate_rollback_images( + value: Mapping[str, Mapping[str, str]], + *, + candidate_image_id: str, +) -> dict[str, dict[str, str]]: + if set(value) != set(READER_SERVICES): + raise ReleasePreparationError("rollback images must cover exactly the reader services") + result: dict[str, dict[str, str]] = {} + for service in READER_SERVICES: + item = value[service] + if not isinstance(item, Mapping) or set(item) != {"image_reference", "image_id"}: + raise ReleasePreparationError("rollback image entry is invalid") + image_reference = str(item["image_reference"] or "").strip() + image_id = str(item["image_id"] or "").strip() + if not _IMAGE_REFERENCE.fullmatch(image_reference): + raise ReleasePreparationError("rollback image reference is not a canonical reader tag") + if not _IMAGE_ID.fullmatch(image_id): + raise ReleasePreparationError("rollback image ID must be an immutable sha256 digest") + if image_id == candidate_image_id: + raise ReleasePreparationError("rollback image must differ from candidate image") + result[service] = {"image_reference": image_reference, "image_id": image_id} + return result + + +def _parse_rollback_role(value: str) -> tuple[str, dict[str, str]]: + service, separator, selector = value.partition("=") + image_reference, at, image_id = selector.rpartition("@") + if not separator or not at or not service or not image_reference or not image_id: + raise ReleasePreparationError( + "rollback role must be service=qdl-v2-python:@sha256:" + ) + if service not in READER_SERVICES: + raise ReleasePreparationError("rollback role is outside the reader services") + return service, {"image_reference": image_reference, "image_id": image_id} + + +def _write_json(path: Path, value: Mapping[str, Any]) -> bytes: + rendered = json.dumps(value, indent=2, sort_keys=True).encode("utf-8") + b"\n" + path.write_bytes(rendered) + path.chmod(_PRIVATE_ARTIFACT_MODE) + return rendered + + +def prepare_alpha_reader_release( + *, + inventory_path: Path, + bindings_dir: Path, + report_path: Path, + output_dir: Path, + source_revision: str, + image_reference: str, + image_id: str, + rollback_images: Mapping[str, Mapping[str, str]], + apply: bool, +) -> dict[str, object]: + """Validate and atomically seal only public alpha-reader release artifacts.""" + + if not _SOURCE_REVISION.fullmatch(source_revision): + raise ReleasePreparationError("source revision must be a full lowercase Git SHA") + if not _IMAGE_REFERENCE.fullmatch(image_reference): + raise ReleasePreparationError("image reference is not a canonical reader tag") + if not _IMAGE_ID.fullmatch(image_id): + raise ReleasePreparationError("candidate image ID must be an immutable sha256 digest") + candidate_images = { + service: {"image_reference": image_reference, "image_id": image_id} + for service in READER_SERVICES + } + validated_rollback_images = _validate_rollback_images( + rollback_images, + candidate_image_id=image_id, + ) + + inventory, inventory_sha256 = _validate_inventory(inventory_path) + report = _validate_report(report_path, inventory_sha256=inventory_sha256) + binding_files = _validate_binding_files(bindings_dir, _binding_digests(report)) + output = _validate_output( + output_dir, + inventory=inventory_path, + bindings=bindings_dir, + report=report_path, + ) + input_hashes = { + "inventory.json": _sha256_bytes(inventory_path.read_bytes()), + "compilation-report.json": _sha256_bytes(report_path.read_bytes()), + **{ + f"bindings/{path.name}": _sha256_bytes(path.read_bytes()) + for path in binding_files + }, + } + manifest_without_digest: dict[str, object] = { + "schema": SCHEMA, + "source_revision": source_revision, + "image_reference": image_reference, + "image_id": image_id, + "image_selector": _image_selector(image_reference, image_id), + "rollback_images": validated_rollback_images, + "services": list(READER_SERVICES), + "inventory_sha256": inventory_sha256, + "compilation_sha256": report["compilation_sha256"], + "binding_count": len(binding_files), + "rollback_services": list(READER_SERVICES), + "input_sha256": input_hashes, + "secret_values_recorded": False, + "runtime_mutations": 0, + "order_actions": 0, + } + manifest = {**manifest_without_digest, "manifest_sha256": _digest(manifest_without_digest)} + result: dict[str, object] = { + "status": "APPLIED" if apply else "DRY_RUN", + "output_dir": str(output), + "manifest_sha256": manifest["manifest_sha256"], + "binding_count": len(binding_files), + "runtime_mutations": 0, + "order_actions": 0, + "secret_values_recorded": False, + } + if not apply: + return result + + staging = output.parent / f".{output.name}.staging-{uuid.uuid4().hex}" + old_umask = os.umask(0o077) + try: + staging.mkdir(mode=0o700) + (staging / "bindings").mkdir(mode=0o750) + (staging / "inventory.json").write_bytes(inventory_path.read_bytes()) + (staging / "inventory.json").chmod(_PRIVATE_ARTIFACT_MODE) + (staging / "compilation-report.json").write_bytes(report_path.read_bytes()) + (staging / "compilation-report.json").chmod(_PRIVATE_ARTIFACT_MODE) + for source in binding_files: + destination = staging / "bindings" / source.name + shutil.copyfile(source, destination) + destination.chmod(_PUBLIC_BINDING_MODE) + override = staging / "reader-image.override.yml" + override.write_bytes(_render_override(candidate_images)) + override.chmod(_PRIVATE_ARTIFACT_MODE) + rollback_override = staging / "reader-rollback.override.yml" + rollback_override.write_bytes(_render_override(validated_rollback_images)) + rollback_override.chmod(_PRIVATE_ARTIFACT_MODE) + _write_json(staging / "release-manifest.json", manifest) + staging.rename(output) + except Exception: + shutil.rmtree(staging, ignore_errors=True) + raise + finally: + os.umask(old_umask) + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--inventory", type=Path, required=True) + parser.add_argument("--bindings-dir", type=Path, required=True) + parser.add_argument("--report", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--image-reference", required=True) + parser.add_argument("--image-id", required=True) + parser.add_argument( + "--rollback-role", + action="append", + required=True, + metavar="SERVICE=IMAGE@SHA256", + help="repeat exactly once for each reader service", + ) + parser.add_argument("--apply", action="store_true") + parser.add_argument("--confirm") + args = parser.parse_args(argv) + if args.apply and args.confirm != CONFIRM: + raise RuntimeError(f"--apply requires --confirm {CONFIRM}") + rollback_images: dict[str, dict[str, str]] = {} + for raw in args.rollback_role: + service, image = _parse_rollback_role(raw) + if service in rollback_images: + raise ReleasePreparationError("rollback role is duplicated") + rollback_images[service] = image + result = prepare_alpha_reader_release( + inventory_path=args.inventory, + bindings_dir=args.bindings_dir, + report_path=args.report, + output_dir=args.output_dir, + source_revision=args.source_revision, + image_reference=args.image_reference, + image_id=args.image_id, + rollback_images=rollback_images, + apply=args.apply, + ) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/rebuild_v2_stable_projection_cache.py b/scripts/rebuild_v2_stable_projection_cache.py index 3593ac6b..9233bf59 100755 --- a/scripts/rebuild_v2_stable_projection_cache.py +++ b/scripts/rebuild_v2_stable_projection_cache.py @@ -24,14 +24,28 @@ KAFKA_BOOTSTRAP = "kafka1:9092,kafka2:9092,kafka3:9092" KAFKA_ADMIN_CONFIG = "/etc/kafka/secrets/admin.properties" EXPECTED_CANONICAL_PARTITIONS = 6 -MAX_ACCEPTED_LAG = 250 +# A live Kafka group has one tail per canonical partition. Preserve the +# original 250-record guard for each partition, then cap the aggregate tail +# separately so a healthy multi-partition stream is not rejected merely for +# having more than one active partition. +MAX_ACCEPTED_LAG = 500 +MAX_ACCEPTED_PARTITION_LAG = 250 REPLAY_LOOKBACK_SECONDS = 15 * 60 MAX_REPLAY_BOOTSTRAP_RECORDS = 1_000_000 REQUIRED_BOUNDED_LAG_SAMPLES = 3 PROJECTOR_SERVICES = ("projector_v2", "projector_v2_2", "projector_v2_3") STREAM_SERVICES = ("stream_v2_active", "stream_v2_passive") QUERY_SERVICES = ("query_v2_1", "query_v2_2") -STOP_SERVICES = (*PROJECTOR_SERVICES, *QUERY_SERVICES, *STREAM_SERVICES) +# A BAR edge checkpoint alone cannot prove that a rebuilt durable cache still +# holds its historical warmup. Stop it with the cache readers so its existing +# startup coverage check can refill only missing final BARs after recovery. +BAR_EDGE_SERVICES = ("binance_bar_edge",) +STOP_SERVICES = ( + *PROJECTOR_SERVICES, + *QUERY_SERVICES, + *STREAM_SERVICES, + *BAR_EDGE_SERVICES, +) CACHE_FILES = ( "/var/lib/qdl-stable/shared/canonical-cache.sqlite3", "/var/lib/qdl-stable/shared/canonical-cache.sqlite3-wal", @@ -88,11 +102,13 @@ def rebuild_plan(env_file: Path) -> dict[str, object]: "lag_gate": { "expected_partitions": EXPECTED_CANONICAL_PARTITIONS, "max_total_records": MAX_ACCEPTED_LAG, + "max_per_partition_records": MAX_ACCEPTED_PARTITION_LAG, "consecutive_samples": REQUIRED_BOUNDED_LAG_SAMPLES, }, "start_order": [ list(STREAM_SERVICES), list(PROJECTOR_SERVICES), + list(BAR_EDGE_SERVICES), list(QUERY_SERVICES), ], "touches_v1": False, @@ -107,8 +123,8 @@ def require_authorization(*, apply: bool, confirm: str | None) -> None: raise ValueError(f"--confirm must equal {CONFIRM_TOKEN}") -def parse_canonical_lag(output: str) -> tuple[int, int]: - lags: list[int] = [] +def parse_canonical_lag(output: str) -> tuple[int, int, int]: + lags: dict[int, int] = {} for line in output.splitlines(): fields = line.split() if len(fields) < 6 or CANONICAL_TOPIC not in fields: @@ -119,10 +135,13 @@ def parse_canonical_lag(output: str) -> tuple[int, int]: partition = fields[topic_index + 1] lag = fields[topic_index + 4] if partition.isdigit() and lag.lstrip("-").isdigit(): - lags.append(max(0, int(lag))) + partition_id = int(partition) + if partition_id in lags: + raise RuntimeError("canonical projector lag output repeats a partition") + lags[partition_id] = max(0, int(lag)) if not lags: raise RuntimeError("canonical projector lag output has no partitions") - return sum(lags), len(lags) + return sum(lags.values()), len(lags), max(lags.values()) def _run( @@ -197,7 +216,7 @@ def _reset_projector_to_bounded_window( "--topic", CANONICAL_TOPIC, "--reset-offsets", "--to-datetime", start_text, "--execute", ) - total_records, partitions = parse_canonical_lag(_kafka_group( + total_records, partitions, _max_partition_lag = parse_canonical_lag(_kafka_group( env_file, "--group", PROJECTOR_GROUP, "--describe" )) if partitions != EXPECTED_CANONICAL_PARTITIONS: @@ -348,16 +367,22 @@ def _wait_projector_ready(env_file: Path, deadline: float) -> None: ) -def lag_sample_acceptable(total_lag: int, partitions: int) -> bool: +def lag_sample_acceptable( + total_lag: int, + partitions: int, + max_partition_lag: int, +) -> bool: return ( partitions == EXPECTED_CANONICAL_PARTITIONS and 0 <= total_lag <= MAX_ACCEPTED_LAG + and 0 <= max_partition_lag <= MAX_ACCEPTED_PARTITION_LAG ) def _wait_bounded_lag(env_file: Path, deadline: float) -> dict[str, int]: last_lag: int | None = None partitions = 0 + max_partition_lag: int | None = None consecutive = 0 observed_bound = 0 while time.monotonic() < deadline: @@ -367,8 +392,8 @@ def _wait_bounded_lag(env_file: Path, deadline: float) -> dict[str, int]: PROJECTOR_GROUP, "--describe", ) - last_lag, partitions = parse_canonical_lag(output) - if lag_sample_acceptable(last_lag, partitions): + last_lag, partitions, max_partition_lag = parse_canonical_lag(output) + if lag_sample_acceptable(last_lag, partitions, max_partition_lag): consecutive += 1 observed_bound = max(observed_bound, last_lag) if consecutive >= REQUIRED_BOUNDED_LAG_SAMPLES: @@ -377,6 +402,8 @@ def _wait_bounded_lag(env_file: Path, deadline: float) -> dict[str, int]: "partitions": partitions, "observed_bound": observed_bound, "configured_bound": MAX_ACCEPTED_LAG, + "configured_per_partition_bound": MAX_ACCEPTED_PARTITION_LAG, + "max_partition_lag": max_partition_lag, "consecutive_samples": consecutive, } else: @@ -385,7 +412,8 @@ def _wait_bounded_lag(env_file: Path, deadline: float) -> dict[str, int]: time.sleep(2) raise TimeoutError( "stable projector did not enter its bounded live-lag window; " - f"last_lag={last_lag} partitions={partitions}" + f"last_lag={last_lag} partitions={partitions} " + f"max_partition_lag={max_partition_lag}" ) @@ -453,6 +481,7 @@ def execute_rebuild(env_file: Path, *, timeout_seconds: float) -> dict[str, obje _start_services(env_file, *PROJECTOR_SERVICES) catchup_deadline = time.monotonic() + timeout_seconds + _start_services(env_file, *BAR_EDGE_SERVICES) lag = _wait_bounded_lag(env_file, catchup_deadline) _wait_projector_ready(env_file, catchup_deadline) @@ -498,7 +527,7 @@ def main() -> int: parser.add_argument("--env-file", type=Path, required=True) parser.add_argument("--apply", action="store_true") parser.add_argument("--confirm") - parser.add_argument("--timeout-seconds", type=float, default=900) + parser.add_argument("--timeout-seconds", type=float, default=1200) args = parser.parse_args() if args.timeout_seconds <= 0: parser.error("--timeout-seconds must be positive") diff --git a/scripts/refresh_v2_l2_core_runtime.py b/scripts/refresh_v2_l2_core_runtime.py index 4ae4474c..4d9207d0 100644 --- a/scripts/refresh_v2_l2_core_runtime.py +++ b/scripts/refresh_v2_l2_core_runtime.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -"""Materialize the declared additive L2 mappings in the active Rust cores. - -This is deliberately narrower than a generic stable-runtime refresh. It -updates only the three mounted realtime-core JSON files and only admits the -six already-declared perpetual BOOK mappings required to complete the -five-liquid V2 demand scope. Existing binding semantics must remain -byte-for-byte equivalent; the current catalog revision is deliberately -materialized for every retained binding so raw/core lineage remains valid. +"""Materialize declared execution-L2 mappings in the active Rust cores. + +This is deliberately narrower than a generic stable-runtime refresh. It +updates only the three mounted realtime-core JSON files and derives additive +BOOK bindings from the execution-grade demand contract. Existing binding +semantics must remain byte-for-byte equivalent; the current catalog revision +is deliberately materialized for every retained binding so raw/core lineage +remains valid. """ from __future__ import annotations @@ -33,19 +33,16 @@ validate_shared_authority_record, write_stable_runtime_bundle, ) +from qdl.runtime.execution_l2 import ( + ExecutionL2MaterializationPlan, + execution_l2_materialization_plan, +) CONFIRM = "REFRESH_QDL_V2_L2_CORE_RUNTIME" DEFAULT_STATE_ROOT = Path("/home/bobby/.local/state/qdl-v2") CORE_FILES = ("core.json", "core-002.json", "core-003.json") -DECLARED_ADDITIVE_BOOK_SOURCE_IDS = frozenset({ - "binance-usdm-bnbusdt-book-primary-v2", - "binance-usdm-dogeusdt-book-primary-v2", - "binance-usdm-solusdt-book-primary-v2", - "okx-swap-bnb-usdt-swap-book-primary-v2", - "okx-swap-doge-usdt-swap-book-primary-v2", - "okx-swap-sol-usdt-swap-book-primary-v2", -}) +DEFAULT_EXECUTION_DEMAND = ROOT / "config/v2/stable-crypto-demand.yaml" _COMMON_METADATA_DRIFT = frozenset({"instrument_catalog_revision"}) @@ -95,11 +92,9 @@ def _without_catalog_revision(value: Mapping[str, Any]) -> dict[str, Any]: def _render_expected( *, authority: Mapping[str, Any], - catalog_path: Path, - acquisition_path: Path, + catalog: StableSourceCatalog, + acquisition: StableAcquisitionPlan, ) -> dict[str, dict[str, Any]]: - catalog = StableSourceCatalog.load(catalog_path) - acquisition = StableAcquisitionPlan.load(acquisition_path, catalog=catalog) with tempfile.TemporaryDirectory(prefix="qdl-l2-core-render-") as raw: runtime = Path(raw) / "runtime" write_stable_runtime_bundle( @@ -119,6 +114,7 @@ def _validate_and_render( active: Mapping[str, Any], expected: Mapping[str, Any], file_name: str, + execution_l2: ExecutionL2MaterializationPlan, ) -> tuple[dict[str, Any], dict[str, Any]]: if active.get("authority") != expected.get("authority"): raise ValueError(f"{file_name} changes authority") @@ -145,15 +141,16 @@ def _validate_and_render( if _without_catalog_revision(current) != _without_catalog_revision(generated): raise ValueError(f"{file_name} has semantic drift for {source_id}") + declared_book_source_ids = frozenset(execution_l2.source_ids) added_ids = expected_by_id.keys() - active_by_id.keys() - if added_ids and added_ids != DECLARED_ADDITIVE_BOOK_SOURCE_IDS: + if added_ids and not added_ids.issubset(declared_book_source_ids): raise ValueError( - f"{file_name} additive BOOK scope differs from the approved declaration: " + f"{file_name} additive BOOK scope differs from the execution demand: " f"{sorted(added_ids)}" ) - if not DECLARED_ADDITIVE_BOOK_SOURCE_IDS.issubset(expected_by_id): + if not declared_book_source_ids.issubset(expected_by_id): raise ValueError(f"{file_name} generated BOOK scope is incomplete") - if not DECLARED_ADDITIVE_BOOK_SOURCE_IDS.issubset(expected_by_id.keys() | active_by_id.keys()): + if not declared_book_source_ids.issubset(expected_by_id.keys() | active_by_id.keys()): raise ValueError(f"{file_name} active/generated BOOK scope is incomplete") additions = [ expected_by_id[str(item["source_id"])] @@ -198,7 +195,7 @@ def _validate_and_render( "after_catalog_revisions": sorted({ int(item["instrument_catalog_revision"]) for item in expected_bindings }), - "declared_book_source_ids": sorted(DECLARED_ADDITIVE_BOOK_SOURCE_IDS), + "declared_book_source_ids": sorted(declared_book_source_ids), "added_book_source_ids": sorted(added_ids), "added_book_symbols": sorted(str(item["native_symbol"]) for item in additions), } @@ -229,6 +226,7 @@ def refresh( apply: bool, catalog_path: Path = ROOT / "config/v2/stable-source-bindings.yaml", acquisition_path: Path = ROOT / "config/v2/stable-acquisition-bindings.yaml", + execution_demand_path: Path = DEFAULT_EXECUTION_DEMAND, state_root: Path = DEFAULT_STATE_ROOT, ) -> dict[str, Any]: runtime_dir = runtime_dir.resolve() @@ -249,10 +247,17 @@ def refresh( validate_shared_authority_record(authority) except ValueError as error: raise ValueError("active authority is invalid") from error + catalog = StableSourceCatalog.load(catalog_path) + acquisition = StableAcquisitionPlan.load(acquisition_path, catalog=catalog) + execution_l2 = execution_l2_materialization_plan( + demand_path=execution_demand_path, + catalog=catalog, + acquisition=acquisition, + ) expected_by_file = _render_expected( authority=authority, - catalog_path=catalog_path, - acquisition_path=acquisition_path, + catalog=catalog, + acquisition=acquisition, ) pending: dict[str, tuple[Path, bytes, bytes, int, dict[str, Any]]] = {} @@ -263,6 +268,7 @@ def refresh( active=_read_json(path, field=f"active {file_name}"), expected=expected_by_file[file_name], file_name=file_name, + execution_l2=execution_l2, ) pending[file_name] = ( path, @@ -290,6 +296,7 @@ def refresh( "runtime_dir": str(runtime_dir), "authority_sha256": _sha256(authority_bytes), "authority_bytes_preserved": True, + "execution_l2": execution_l2.evidence(), "files": files, "production_mutations": 0 if not apply else len(CORE_FILES), } @@ -327,6 +334,7 @@ def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--runtime-dir", type=Path, required=True) parser.add_argument("--output-dir", type=Path) + parser.add_argument("--execution-demand", type=Path, default=DEFAULT_EXECUTION_DEMAND) parser.add_argument("--apply", action="store_true") parser.add_argument("--confirm") args = parser.parse_args(argv) @@ -336,6 +344,7 @@ def main(argv: list[str] | None = None) -> int: runtime_dir=args.runtime_dir, output_dir=args.output_dir, apply=args.apply, + execution_demand_path=args.execution_demand, ), indent=2, sort_keys=True)) return 0 diff --git a/scripts/refresh_v2_rust_core_runtime.py b/scripts/refresh_v2_rust_core_runtime.py index 19f812a3..8d6e14fd 100644 --- a/scripts/refresh_v2_rust_core_runtime.py +++ b/scripts/refresh_v2_rust_core_runtime.py @@ -4,7 +4,7 @@ The stable successor runtime is a role-config directory, not a generic bundle. This tool deliberately rewrites only ``core.json``, ``core-002.json`` and ``core-003.json`` after proving that their sole semantic change is the declared -L2 snapshot refresh cadence. It also advances only the immutable Rust image +L2 materialized-snapshot cadence. It also advances only the immutable Rust image selector in the sibling Compose environment file. All prior bytes are kept in one private rollback directory before the atomic replacements occur. """ @@ -30,16 +30,23 @@ from qdl.runtime.stable_catalog import StableSourceCatalog from qdl.runtime.stable_deployment import ( + STABLE_CORE_DEDUP_CAPACITY, StableAcquisitionPlan, validate_shared_authority_record, write_stable_runtime_bundle, ) +from qdl.runtime.execution_l2 import ( + ExecutionL2MaterializationPlan, + execution_l2_materialization_plan, +) CONFIRM = "REFRESH_QDL_V2_RUST_CORE_RUNTIME" DEFAULT_STATE_ROOT = Path("/home/bobby/.local/state/qdl-v2") CORE_FILES = ("core.json", "core-002.json", "core-003.json") _DIGEST = re.compile(r"sha256:[0-9a-f]{64}\Z") +DEFAULT_EXECUTION_DEMAND = ROOT / "config/v2/stable-crypto-demand.yaml" +_PREVIOUS_STABLE_CORE_DEDUP_CAPACITY = 1_000_000 def _sha256(value: bytes) -> str: @@ -93,25 +100,39 @@ def _bindings(value: Mapping[str, Any], *, field: str) -> tuple[dict[str, Any], return tuple(result) -def _without_bindings(value: Mapping[str, Any]) -> dict[str, Any]: +def _without_bindings_and_dedup_capacity(value: Mapping[str, Any]) -> tuple[dict[str, Any], int]: result = dict(value) core = result.get("core") if not isinstance(core, Mapping): raise ValueError("core config lacks core object") normalized_core = dict(core) normalized_core.pop("bindings", None) + dedup_capacity = normalized_core.pop("dedup_capacity", None) + if not isinstance(dedup_capacity, int): + raise ValueError("core config lacks integer dedup capacity") result["core"] = normalized_core - return result + return result, dedup_capacity -def _validate_only_snapshot_refresh( +def _validate_only_materialized_snapshot_interval( active: Mapping[str, Any], expected: Mapping[str, Any], *, file_name: str, + execution_l2: ExecutionL2MaterializationPlan, ) -> dict[str, Any]: - if _without_bindings(active) != _without_bindings(expected): + active_without, active_dedup_capacity = _without_bindings_and_dedup_capacity(active) + expected_without, expected_dedup_capacity = _without_bindings_and_dedup_capacity(expected) + if active_without != expected_without: raise ValueError(f"{file_name} changes a non-binding core field") + if ( + active_dedup_capacity not in { + _PREVIOUS_STABLE_CORE_DEDUP_CAPACITY, + STABLE_CORE_DEDUP_CAPACITY, + } + or expected_dedup_capacity != STABLE_CORE_DEDUP_CAPACITY + ): + raise ValueError(f"{file_name} has an invalid bounded dedup transition") before = _bindings(active, field=f"active {file_name}") after = _bindings(expected, field=f"expected {file_name}") before_ids = [str(item["source_id"]) for item in before] @@ -120,6 +141,7 @@ def _validate_only_snapshot_refresh( raise ValueError(f"{file_name} changes binding order or membership") l2_sources: list[str] = [] + declared_source_ids = frozenset(execution_l2.source_ids) for current, refreshed in zip(before, after, strict=True): source_id = str(current["source_id"]) if current.get("l2") is None: @@ -134,28 +156,54 @@ def _validate_only_snapshot_refresh( raise ValueError(f"{file_name} has invalid L2 contract: {source_id}") current_l2 = dict(current_l2) refreshed_l2 = dict(refreshed_l2) - old_cadence = current_l2.pop("snapshot_refresh_seconds", None) - new_cadence = refreshed_l2.pop("snapshot_refresh_seconds", None) + old_provider_refresh = current_l2.pop("snapshot_refresh_seconds", None) + new_provider_refresh = refreshed_l2.pop("snapshot_refresh_seconds", None) + old_materialization = current_l2.pop("materialized_snapshot_interval_ms", None) + new_materialization = refreshed_l2.pop("materialized_snapshot_interval_ms", None) current_without["l2"] = current_l2 refreshed_without["l2"] = refreshed_l2 if current_without != refreshed_without: raise ValueError(f"{file_name} changes L2 binding fields beyond cadence: {source_id}") - if old_cadence is not None or new_cadence != 30: - raise ValueError(f"{file_name} has invalid L2 cadence transition: {source_id}") + if old_provider_refresh != new_provider_refresh or new_provider_refresh != 30: + raise ValueError(f"{file_name} changes provider refresh cadence: {source_id}") + if source_id in declared_source_ids: + if ( + old_materialization not in { + None, + execution_l2.materialized_snapshot_interval_ms, + } + or new_materialization != execution_l2.materialized_snapshot_interval_ms + ): + raise ValueError( + f"{file_name} has invalid execution L2 materialization transition: {source_id}" + ) + elif old_materialization != new_materialization: + raise ValueError( + f"{file_name} changes non-execution L2 materialization cadence: {source_id}" + ) l2_sources.append(source_id) if not l2_sources: raise ValueError(f"{file_name} has no L2 binding to refresh") - return {"file": file_name, "l2_source_ids": sorted(l2_sources)} + if not declared_source_ids.issubset(l2_sources): + raise ValueError(f"{file_name} lacks a declared execution L2 binding") + return { + "file": file_name, + "l2_source_ids": sorted(l2_sources), + "materialized_snapshot_interval_ms": execution_l2.materialized_snapshot_interval_ms, + "execution_l2_source_ids": sorted(declared_source_ids), + "dedup_capacity": { + "before": active_dedup_capacity, + "after": expected_dedup_capacity, + }, + } def _render_expected( *, authority: Mapping[str, Any], - catalog_path: Path, - acquisition_path: Path, + catalog: StableSourceCatalog, + acquisition: StableAcquisitionPlan, ) -> dict[str, bytes]: - catalog = StableSourceCatalog.load(catalog_path) - acquisition = StableAcquisitionPlan.load(acquisition_path, catalog=catalog) with tempfile.TemporaryDirectory(prefix="qdl-core-runtime-render-") as raw: runtime = Path(raw) / "runtime" write_stable_runtime_bundle( @@ -214,6 +262,7 @@ def refresh( apply: bool, catalog_path: Path = ROOT / "config/v2/stable-source-bindings.yaml", acquisition_path: Path = ROOT / "config/v2/stable-acquisition-bindings.yaml", + execution_demand_path: Path = DEFAULT_EXECUTION_DEMAND, state_root: Path = DEFAULT_STATE_ROOT, ) -> dict[str, Any]: runtime_dir = runtime_dir.resolve() @@ -225,8 +274,6 @@ def refresh( raise ValueError("active rollout environment is invalid") if _DIGEST.fullmatch(active_rust_image) is None or _DIGEST.fullmatch(new_rust_image) is None: raise ValueError("Rust image selectors must be immutable SHA-256 digests") - if active_rust_image == new_rust_image: - raise ValueError("Rust core runtime refresh requires a different immutable image") if apply: if output_dir is None: raise ValueError("apply requires an output directory") @@ -234,18 +281,26 @@ def refresh( if output_dir.exists() or state_root not in (output_dir, *output_dir.parents): raise ValueError("output directory must be a new private QDL state path") + catalog = StableSourceCatalog.load(catalog_path) + acquisition = StableAcquisitionPlan.load(acquisition_path, catalog=catalog) + execution_l2 = execution_l2_materialization_plan( + demand_path=execution_demand_path, + catalog=catalog, + acquisition=acquisition, + ) authority, authority_bytes = _load_authority(runtime_dir) expected_bytes = _render_expected( authority=authority, - catalog_path=catalog_path, - acquisition_path=acquisition_path, + catalog=catalog, + acquisition=acquisition, ) active_bytes = {name: (runtime_dir / name).read_bytes() for name in CORE_FILES} changes = [ - _validate_only_snapshot_refresh( + _validate_only_materialized_snapshot_interval( _read_json(runtime_dir / name, field=f"active {name}"), json.loads(expected_bytes[name]), file_name=name, + execution_l2=execution_l2, ) for name in CORE_FILES ] @@ -253,6 +308,7 @@ def refresh( next_env = _replace_rust_image( active_env, expected_old=active_rust_image, new=new_rust_image ) + image_selector_changed = active_env != next_env result: dict[str, Any] = { "schema": "qdl.v2.rust-core-runtime-refresh.v1", "status": "APPLIED" if apply else "DRY_RUN", @@ -262,13 +318,15 @@ def refresh( "authority_bytes_preserved": True, "active_rust_image": active_rust_image, "new_rust_image": new_rust_image, + "image_selector_changed": image_selector_changed, + "execution_l2": execution_l2.evidence(), "changes": changes, "files": { name: {"before": _sha256(active_bytes[name]), "after": _sha256(expected_bytes[name])} for name in CORE_FILES }, "rollout_env_sha256": {"before": _sha256(active_env), "after": _sha256(next_env)}, - "production_mutations": 0 if not apply else 4, + "production_mutations": 0 if not apply else len(CORE_FILES) + int(image_selector_changed), } if not apply: return result @@ -292,8 +350,9 @@ def refresh( target = runtime_dir / name _write_atomic(target, expected_bytes[name], mode=0o644) applied.append(target) - _write_atomic(rollout_env, next_env, mode=0o600) - applied.append(rollout_env) + if image_selector_changed: + _write_atomic(rollout_env, next_env, mode=0o600) + applied.append(rollout_env) except Exception: for target in reversed(applied): backup = rollback_dir / ("rollout.env" if target == rollout_env else target.name) @@ -317,6 +376,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--rollout-env", type=Path, required=True) parser.add_argument("--active-rust-image", required=True) parser.add_argument("--new-rust-image", required=True) + parser.add_argument("--execution-demand", type=Path, default=DEFAULT_EXECUTION_DEMAND) parser.add_argument("--output-dir", type=Path) parser.add_argument("--apply", action="store_true") parser.add_argument("--confirm") @@ -330,6 +390,7 @@ def main(argv: list[str] | None = None) -> int: new_rust_image=args.new_rust_image, output_dir=args.output_dir, apply=args.apply, + execution_demand_path=args.execution_demand, ) print(json.dumps(result, indent=2, sort_keys=True)) return 0 diff --git a/scripts/repair_stable_final_bar_history.py b/scripts/repair_stable_final_bar_history.py new file mode 100644 index 00000000..cdcd60a3 --- /dev/null +++ b/scripts/repair_stable_final_bar_history.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Repair a bounded final-BAR history hole through the normal V2 data plane.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from qdl.runtime.stable_bar_edge import build_from_environment + + +CONFIRM = "REPAIR_QDL_STABLE_FINAL_BAR_HISTORY" + + +def _expected_missing(values: list[str]) -> dict[str, int]: + result: dict[str, int] = {} + for value in values: + binding_id, separator, count = value.partition("=") + if not separator or not binding_id or not count.isdigit(): + raise ValueError("--expected-missing must use binding_id=non_negative_integer") + if binding_id in result: + raise ValueError("--expected-missing binding appears more than once") + result[binding_id] = int(count) + return result + + +def _summary(plan, *, remaining_rows: int | None = None, published_rows: int | None = None) -> dict: + result = { + "binding_id": plan.source.binding_id, + "venue": plan.acquisition.runtime, + "window_rows": len(plan.envelopes), + "missing_rows": len(plan.missing_envelopes), + "first_open_ms": min(plan.expected_opens), + "last_open_ms": max(plan.expected_opens), + } + if published_rows is not None: + result["published_rows"] = published_rows + if remaining_rows is not None: + result["remaining_rows"] = remaining_rows + return result + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--binding", action="append", required=True) + parser.add_argument("--rows", type=int, required=True) + parser.add_argument("--expected-missing", action="append", required=True) + parser.add_argument("--wait-seconds", type=float, default=180.0) + parser.add_argument("--poll-seconds", type=float, default=1.0) + parser.add_argument("--apply", action="store_true") + parser.add_argument("--confirm") + args = parser.parse_args(argv) + + bindings = tuple(dict.fromkeys(args.binding)) + if len(bindings) != len(args.binding): + raise SystemExit("--binding may not repeat") + if not 1 <= args.rows <= 10_000: + raise SystemExit("--rows must be between 1 and 10000") + if args.wait_seconds <= 0 or args.poll_seconds <= 0: + raise SystemExit("wait and poll durations must be positive") + try: + expected = _expected_missing(args.expected_missing) + except ValueError as error: + raise SystemExit(str(error)) from error + if set(expected) != set(bindings): + raise SystemExit("--expected-missing must name exactly the requested bindings") + if args.apply and args.confirm != CONFIRM: + raise SystemExit(f"--apply requires --confirm {CONFIRM}") + + edge = build_from_environment( + client_id=f"qdl-v2-final-bar-repair-{os.getpid()}", + repair_only=True, + ) + try: + plans = tuple( + edge.prepare_history_repair(binding_id, rows=args.rows) + for binding_id in bindings + ) + for plan in plans: + actual = len(plan.missing_envelopes) + required = expected[plan.source.binding_id] + if actual != required: + raise RuntimeError( + "stable BAR repair missing-row count differs from approved scope " + f"binding={plan.source.binding_id} expected={required} actual={actual}" + ) + if not args.apply: + print(json.dumps({ + "schema": "qdl.stable-final-bar-history-repair.v1", + "status": "DRY_RUN", + "production_mutations": 0, + "repairs": [_summary(plan) for plan in plans], + }, sort_keys=True)) + return 0 + + published = { + plan.source.binding_id: edge.apply_history_repair( + plan, + expected_missing_rows=expected[plan.source.binding_id], + ) + for plan in plans + } + deadline = time.monotonic() + args.wait_seconds + remaining = {plan.source.binding_id: -1 for plan in plans} + while time.monotonic() < deadline: + remaining = { + plan.source.binding_id: edge.history_repair_remaining_rows(plan) + for plan in plans + } + if all(value == 0 for value in remaining.values()): + print(json.dumps({ + "schema": "qdl.stable-final-bar-history-repair.v1", + "status": "CONVERGED", + "production_mutations": sum(published.values()), + "repairs": [ + _summary( + plan, + published_rows=published[plan.source.binding_id], + remaining_rows=remaining[plan.source.binding_id], + ) + for plan in plans + ], + }, sort_keys=True)) + return 0 + time.sleep(args.poll_seconds) + raise RuntimeError( + "stable BAR repair did not converge before deadline remaining=" + + json.dumps(remaining, sort_keys=True) + ) + finally: + edge.stop() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_alpha_deployment_bindings.py b/tests/test_alpha_deployment_bindings.py new file mode 100644 index 00000000..92d5ecee --- /dev/null +++ b/tests/test_alpha_deployment_bindings.py @@ -0,0 +1,372 @@ +from __future__ import annotations + +from copy import deepcopy +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +import yaml + +from qdl.consumer.universal_release import ConsumerRouteBinding + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = ROOT / "scripts/compile_alpha_deployment_bindings.py" + + +def _module(): + spec = importlib.util.spec_from_file_location("alpha_deployment_bindings", SCRIPT_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load(path: Path): + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +class AlphaDeploymentBindingCompilerTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.tool = _module() + cls.catalog_path = ROOT / "config/v2/stable-source-bindings.yaml" + cls.reference_path = ROOT / "consumers/stable/reference-l2-stable.yaml" + cls.release_path = ROOT / "config/v2/stable-v2-release-routing.yaml" + cls.policy_path = ROOT / "config/v2/universal-release-policy.yaml" + cls.catalog = _load(cls.catalog_path) + cls.reference = _load(cls.reference_path) + cls.release = _load(cls.release_path) + cls.policy = _load(cls.policy_path) + + @staticmethod + def _route( + *, + venue: str, + market: str, + symbol: str, + feed: str, + interval: str | None = None, + source_policy_id: str = "crypto_primary_v2", + provider_plane: str = "REALTIME", + fallback: str = "BLOCKED", + fallback_rule_id: str | None = None, + blocked_reason: str | None = "V1_EQUIVALENCE_UNPROVEN", + freshness: int = 15_000, + final: bool = False, + live: bool = True, + ) -> dict[str, object]: + return { + "venue": venue, + "market": market, + "product_type": "PERPETUAL", + "native_symbol": symbol, + "feed": feed, + "interval": interval, + "source_policy_id": source_policy_id, + "provider_plane": provider_plane, + "max_freshness_ms": freshness, + "require_final_bars": final, + "require_live": live, + "gap_policy": "BLOCK", + "fallback": fallback, + "fallback_rule_id": fallback_rule_id, + "blocked_reason": blocked_reason, + } + + def _deployment( + self, + *, + deployment_id: str, + alpha_id: str, + profile: str, + status: str = "DECLARED", + maxlen: int = 2_500, + min_bars: int = 200, + routes: list[dict[str, object]], + ) -> dict[str, object]: + return { + "deployment_id": deployment_id, + "alpha_id": alpha_id, + "alpha_directory": alpha_id.split("_")[0], + "profile": profile, + "mode": "paper", + "status": status, + "blocked_reason": None, + "compose": {"path": "alphas/example/docker-compose.yml", "service": alpha_id, "sha256": "a" * 64}, + "sources": {"strategy_config_path": "alphas/example/config.yaml", "strategy_config_sha256": "b" * 64}, + "history": {"maxlen": maxlen, "min_bars": min_bars}, + "routes": routes, + } + + def _inventory(self, deployments: list[dict[str, object]]) -> dict[str, object]: + value: dict[str, object] = { + "schema": "execution-alpha.data-requirements.v1", + "revision": 1, + "registry_path": "runtime/config/v2_alpha_deployment_requirements.yaml", + "registry_sha256": "c" * 64, + "deployments": deployments, + } + value["inventory_sha256"] = self.tool._digest(value) + return value + + def _compile(self, inventory: dict[str, object]) -> dict[str, object]: + return self.tool.compile_inventory( + inventory=inventory, + catalog=self.catalog, + reference_manifest=self.reference, + release_routing=self.release, + release_routing_sha256=self.tool._sha256_file(self.release_path), + policy=self.policy, + policy_sha256=self.tool._sha256_file(self.policy_path), + catalog_sha256=self.tool._sha256_file(self.catalog_path), + reference_manifest_sha256=self.tool._sha256_file(self.reference_path), + ) + + def test_compiles_native_binance_and_okx_bindings_with_standard_contract(self) -> None: + binance = self._deployment( + deployment_id="adaptive:binance:paper", + alpha_id="adaptive_hma_cpp_00115m", + profile="directional_bar", + maxlen=2_500, + min_bars=400, + routes=[self._route( + venue="BINANCE", market="USDM", symbol="ETHUSDT", feed="BAR", interval="15m", + freshness=180_000, final=True, live=False, + blocked_reason="V1_FINAL_BAR_EQUIVALENCE_UNPROVEN", + )], + ) + okx = self._deployment( + deployment_id="adaptive:okx:no-order", + alpha_id="adaptive_hma_cpp_00115m", + profile="directional_bar", + status="DECLARED_NO_ORDER_PROBE", + maxlen=5_000, + min_bars=400, + routes=[self._route( + venue="OKX", market="SWAP", symbol="ETH-USDT-SWAP", feed="BAR", interval="15m", + freshness=180_000, final=True, live=False, + blocked_reason="V1_FINAL_BAR_EQUIVALENCE_UNPROVEN", + )], + ) + report = self._compile(self._inventory([binance, okx])) + self.assertEqual( + [item["status"] for item in report["deployments"]], + ["ADMITTED", "ADMITTED"], + msg=str({item["deployment_id"]: item["reason"] for item in report["deployments"]}), + ) + for item in report["deployments"]: + binding = ConsumerRouteBinding.from_canonical_mapping(item["binding"]) + self.assertEqual(len(binding.products), 1) + product = binding.products[0] + self.assertEqual(product.feed, "BAR") + self.assertEqual(product.interval, "15m") + self.assertTrue(product.require_final_bars) + self.assertFalse(product.execution_grade) + self.assertEqual(product.fallback, "BLOCKED") + self.assertEqual(report["deployments"][0]["history"], {"maxlen": 2_500, "min_bars": 400}) + self.assertEqual(report["deployments"][1]["history"], {"maxlen": 5_000, "min_bars": 400}) + + def test_grid_l2_and_binance_basis_reference_are_admitted_without_execution_grade(self) -> None: + grid_routes = [ + self._route( + venue="BINANCE", market="USDM", symbol="ETHUSDT", feed="BAR", interval="1h", + freshness=180_000, final=True, live=False, + blocked_reason="V1_FINAL_BAR_EQUIVALENCE_UNPROVEN", + ), + self._route( + venue="BINANCE", market="USDM", symbol="ETHUSDT", feed="TRADE", + fallback="V1", fallback_rule_id="V1_TRADE_EQUIVALENCE_CERTIFIED", blocked_reason=None, + ), + self._route(venue="BINANCE", market="USDM", symbol="ETHUSDT", feed="QUOTE"), + self._route( + venue="BINANCE", market="USDM", symbol="ETHUSDT", feed="MARK_INDEX_PRICE", + source_policy_id="crypto_liquid_v2", provider_plane="REFERENCE", + ), + self._route( + venue="BINANCE", market="USDM", symbol="ETHUSDT", feed="BOOK_SNAPSHOT", + source_policy_id="crypto_liquid_v2", provider_plane="L2", freshness=60_000, + ), + self._route( + venue="BINANCE", market="USDM", symbol="ETHUSDT", feed="BOOK_DELTA", + source_policy_id="crypto_liquid_v2", provider_plane="L2", freshness=2_000, + ), + ] + basis_routes = [ + self._route( + venue="BINANCE", market="USDM", symbol="BTCUSDT", feed=feed, + interval="1d" if feed in {"OPEN_INTEREST", "LONG_SHORT_RATIO", "TAKER_FLOW", "BASIS"} else None, + source_policy_id="crypto_liquid_v2", provider_plane="REFERENCE", + freshness=86_400_000, live=False, + ) + for feed in ( + "FUNDING_RATE", "OPEN_INTEREST", "LONG_SHORT_RATIO", "TAKER_FLOW", + "MARK_INDEX_PRICE", "CONTRACT_METADATA", "BASIS", + ) + ] + report = self._compile(self._inventory([ + self._deployment( + deployment_id="grid:binance:paper", alpha_id="dynamic_grid_0011h", + profile="grid_l2", maxlen=5_000, min_bars=100, routes=grid_routes, + ), + self._deployment( + deployment_id="basis:binance:paper", alpha_id="basis_arb_binance_0011d", + profile="basis_reference", maxlen=10_000, min_bars=365, routes=basis_routes, + ), + ])) + self.assertEqual( + [item["status"] for item in report["deployments"]], + ["ADMITTED", "ADMITTED"], + msg=str({item["deployment_id"]: item["reason"] for item in report["deployments"]}), + ) + compiled = {item["deployment_id"]: item for item in report["deployments"]} + grid = ConsumerRouteBinding.from_canonical_mapping(compiled["grid:binance:paper"]["binding"]) + self.assertEqual({item.feed for item in grid.products}, { + "BAR", "TRADE", "QUOTE", "MARK_INDEX_PRICE", "BOOK_SNAPSHOT", "BOOK_DELTA", + }) + self.assertTrue(all(not item.execution_grade for item in grid.products)) + basis = ConsumerRouteBinding.from_canonical_mapping(compiled["basis:binance:paper"]["binding"]) + self.assertEqual(len(basis.products), 7) + self.assertEqual({item.provider_plane for item in basis.products}, {"REFERENCE"}) + self.assertEqual(compiled["basis:binance:paper"]["history"]["maxlen"], 10_000) + + def test_missing_or_invalid_required_capability_blocks_whole_deployment(self) -> None: + unsupported_okx_basis = self._deployment( + deployment_id="basis:okx:no-order", alpha_id="basis_arb_binance_0011d", + profile="basis_reference", status="DECLARED_NO_ORDER_PROBE", routes=[self._route( + venue="OKX", market="SWAP", symbol="BTC-USDT-SWAP", feed="BASIS", interval="1d", + source_policy_id="crypto_liquid_v2", provider_plane="REFERENCE", freshness=86_400_000, live=False, + )], + ) + unknown = self._deployment( + deployment_id="unknown:binance:paper", alpha_id="adaptive_hma_cpp_unknown", + profile="directional_bar", routes=[self._route( + venue="BINANCE", market="USDM", symbol="UNKNOWNUSDT", feed="BAR", interval="15m", + freshness=180_000, final=True, live=False, + )], + ) + invalid_fallback = self._deployment( + deployment_id="invalid:binance:paper", alpha_id="dynamic_grid_invalid", + profile="grid_l2", routes=[self._route( + venue="BINANCE", market="USDM", symbol="ETHUSDT", feed="QUOTE", + fallback="V1", fallback_rule_id="INVALID", blocked_reason=None, + )], + ) + out_of_bounds = self._deployment( + deployment_id="long:binance:paper", alpha_id="adaptive_hma_cpp_long", + profile="directional_bar", maxlen=10_001, routes=[self._route( + venue="BINANCE", market="USDM", symbol="ETHUSDT", feed="BAR", interval="15m", + freshness=180_000, final=True, live=False, + )], + ) + report = self._compile(self._inventory([unsupported_okx_basis, unknown, invalid_fallback, out_of_bounds])) + blocked = {item["deployment_id"]: item["reason"] for item in report["deployments"]} + self.assertTrue(blocked["basis:okx:no-order"].startswith("CAPABILITY_UNAVAILABLE:BASIS")) + self.assertTrue(blocked["unknown:binance:paper"].startswith("CATALOG_IDENTITY_UNAVAILABLE")) + self.assertIn("V1 fallback is allowed only", blocked["invalid:binance:paper"]) + self.assertIn("must not exceed 10000", blocked["long:binance:paper"]) + self.assertTrue(all(item["binding"] is None for item in report["deployments"])) + + def test_optional_unavailable_metric_is_explicit_and_never_becomes_zero(self) -> None: + required_bar = self._route( + venue="OKX", market="SWAP", symbol="BTC-USDT-SWAP", feed="BAR", interval="1h", + freshness=180_000, final=True, live=False, + ) + optional_basis = self._route( + venue="OKX", market="SWAP", symbol="BTC-USDT-SWAP", feed="BASIS", interval="1d", + source_policy_id="crypto_liquid_v2", provider_plane="REFERENCE", + freshness=86_400_000, live=False, + ) + optional_basis["required"] = False + deployment = self._deployment( + deployment_id="optional:okx:no-order", alpha_id="basis_arb_binance_0011d", + profile="basis_reference", status="DECLARED_NO_ORDER_PROBE", + routes=[required_bar, optional_basis], + ) + + report = self._compile(self._inventory([deployment])) + rendered = report["deployments"][0] + self.assertEqual(rendered["status"], "ADMITTED") + self.assertEqual(rendered["optional_unavailable"], [{ + "venue": "OKX", + "market": "SWAP", + "product_type": "PERPETUAL", + "native_symbol": "BTC-USDT-SWAP", + "feed": "BASIS", + "interval": "1d", + "source_policy_id": "crypto_liquid_v2", + "reason": "CAPABILITY_UNAVAILABLE:BASIS:crypto_liquid_v2", + }]) + binding = ConsumerRouteBinding.from_canonical_mapping(rendered["binding"]) + self.assertEqual([(product.feed, product.interval) for product in binding.products], [("BAR", "1h")]) + + only_optional = deepcopy(deployment) + only_optional["deployment_id"] = "optional-only:okx:no-order" + only_optional["routes"] = [optional_basis] + no_product = self._compile(self._inventory([only_optional]))["deployments"][0] + self.assertEqual(no_product["status"], "BLOCKED") + self.assertEqual(no_product["reason"], "NO_ADMITTED_REQUIRED_ROUTE") + self.assertEqual(no_product["optional_unavailable"], []) + + def test_rejects_tampered_inventory_and_non_boolean_policy(self) -> None: + deployment = self._deployment( + deployment_id="tampered:binance:paper", alpha_id="adaptive_hma_cpp_tampered", + profile="directional_bar", routes=[self._route( + venue="BINANCE", market="USDM", symbol="ETHUSDT", feed="BAR", interval="15m", + freshness=180_000, final=True, live=False, + )], + ) + inventory = self._inventory([deployment]) + tampered = deepcopy(inventory) + tampered["deployments"][0]["history"]["maxlen"] = 5_000 + with self.assertRaisesRegex(self.tool.DeploymentBindingError, "inventory checksum differs"): + self._compile(tampered) + + invalid = deepcopy(inventory) + invalid["deployments"][0]["routes"][0]["require_final_bars"] = "true" + invalid["inventory_sha256"] = self.tool._digest({ + key: value for key, value in invalid.items() if key != "inventory_sha256" + }) + report = self._compile(invalid) + self.assertEqual(report["deployments"][0]["status"], "BLOCKED") + self.assertIn("must be boolean", report["deployments"][0]["reason"]) + + invalid_required = deepcopy(inventory) + invalid_required["deployments"][0]["routes"][0]["required"] = "false" + invalid_required["inventory_sha256"] = self.tool._digest({ + key: value for key, value in invalid_required.items() if key != "inventory_sha256" + }) + report = self._compile(invalid_required) + self.assertEqual(report["deployments"][0]["status"], "BLOCKED") + self.assertIn("route required must be boolean", report["deployments"][0]["reason"]) + + def test_is_deterministic_and_write_scope_is_named_output_only(self) -> None: + deployment = self._deployment( + deployment_id="adaptive:binance:paper", alpha_id="adaptive_hma_cpp_00115m", + profile="directional_bar", routes=[self._route( + venue="BINANCE", market="USDM", symbol="ETHUSDT", feed="BAR", interval="15m", + freshness=180_000, final=True, live=False, + )], + ) + inventory = self._inventory([deployment]) + first = self._compile(inventory) + second = self._compile(deepcopy(inventory)) + self.assertEqual(first, second) + with tempfile.TemporaryDirectory() as raw: + output = Path(raw) / "bindings" + changed = self.tool.write_compilation(output, first) + self.assertEqual(len(changed), 2) + self.assertEqual(self.tool.write_compilation(output, first), ()) + binding_paths = list(output.glob("*.binding.json")) + self.assertEqual(len(binding_paths), 1) + payload = json.loads(binding_paths[0].read_text(encoding="utf-8")) + self.assertEqual( + ConsumerRouteBinding.from_canonical_mapping(payload).canonical_mapping(), + payload, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_alpha_reader_release.py b/tests/test_alpha_reader_release.py new file mode 100644 index 00000000..95b1ae5f --- /dev/null +++ b/tests/test_alpha_reader_release.py @@ -0,0 +1,288 @@ +from __future__ import annotations + +from copy import deepcopy +import importlib.util +import json +from pathlib import Path +import stat +import tempfile +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +COMPILER_PATH = ROOT / "scripts/compile_alpha_deployment_bindings.py" +PREPARER_PATH = ROOT / "scripts/prepare_alpha_reader_release.py" +IMAGE_ID = "sha256:" + "a" * 64 +QUERY_ROLLBACK_IMAGE_ID = "sha256:" + "b" * 64 +STREAM_ROLLBACK_IMAGE_ID = "sha256:" + "c" * 64 +SOURCE_REVISION = "c" * 40 +ROLLBACK_IMAGES = { + "query_v2_1": { + "image_reference": "qdl-v2-python:2.0.0-rollback-query-bbbbbbb", + "image_id": QUERY_ROLLBACK_IMAGE_ID, + }, + "query_v2_2": { + "image_reference": "qdl-v2-python:2.0.0-rollback-query-bbbbbbb", + "image_id": QUERY_ROLLBACK_IMAGE_ID, + }, + "stream_v2_active": { + "image_reference": "qdl-v2-python:2.0.0-rollback-stream-ccccccc", + "image_id": STREAM_ROLLBACK_IMAGE_ID, + }, + "stream_v2_passive": { + "image_reference": "qdl-v2-python:2.0.0-rollback-stream-ccccccc", + "image_id": STREAM_ROLLBACK_IMAGE_ID, + }, +} + + +def _module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class AlphaReaderReleaseTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.compiler = _module("alpha_deployment_bindings", COMPILER_PATH) + cls.preparer = _module("alpha_reader_release", PREPARER_PATH) + cls.catalog_path = ROOT / "config/v2/stable-source-bindings.yaml" + cls.reference_path = ROOT / "consumers/stable/reference-l2-stable.yaml" + cls.routing_path = ROOT / "config/v2/stable-v2-release-routing.yaml" + cls.policy_path = ROOT / "config/v2/universal-release-policy.yaml" + cls.catalog = yaml.safe_load(cls.catalog_path.read_text(encoding="utf-8")) + cls.reference = yaml.safe_load(cls.reference_path.read_text(encoding="utf-8")) + cls.routing = yaml.safe_load(cls.routing_path.read_text(encoding="utf-8")) + cls.policy = yaml.safe_load(cls.policy_path.read_text(encoding="utf-8")) + + def _compiled_input(self, root: Path) -> tuple[Path, Path, Path]: + root.mkdir(parents=True, exist_ok=True) + route = { + "venue": "BINANCE", + "market": "USDM", + "product_type": "PERPETUAL", + "native_symbol": "ETHUSDT", + "feed": "BAR", + "interval": "15m", + "source_policy_id": "crypto_primary_v2", + "provider_plane": "REALTIME", + "max_freshness_ms": 180000, + "require_final_bars": True, + "require_live": False, + "gap_policy": "BLOCK", + "fallback": "BLOCKED", + "fallback_rule_id": None, + "blocked_reason": "V1_FINAL_BAR_EQUIVALENCE_UNPROVEN", + "required": True, + } + deployment = { + "deployment_id": "adaptive:binance:paper", + "alpha_id": "adaptive_hma_cpp_00115m", + "alpha_directory": "adaptive_hma_cpp", + "profile": "directional_bar", + "mode": "paper", + "status": "DECLARED", + "blocked_reason": None, + "compose": {"path": "alphas/example/docker-compose.yml", "service": "adaptive", "sha256": "a" * 64}, + "sources": {"strategy_config_path": "alphas/example/config.yaml", "strategy_config_sha256": "b" * 64}, + "history": {"maxlen": 2500, "min_bars": 400}, + "routes": [route], + } + inventory: dict[str, object] = { + "schema": "execution-alpha.data-requirements.v1", + "revision": 1, + "registry_path": "runtime/config/v2_alpha_deployment_requirements.yaml", + "registry_sha256": "c" * 64, + "deployments": [deployment], + } + inventory["inventory_sha256"] = self.compiler._digest(inventory) + report = self.compiler.compile_inventory( + inventory=inventory, + catalog=self.catalog, + reference_manifest=self.reference, + release_routing=self.routing, + release_routing_sha256=self.compiler._sha256_file(self.routing_path), + policy=self.policy, + policy_sha256=self.compiler._sha256_file(self.policy_path), + catalog_sha256=self.compiler._sha256_file(self.catalog_path), + reference_manifest_sha256=self.compiler._sha256_file(self.reference_path), + ) + inventory_path = root / "inventory.json" + inventory_path.write_text(json.dumps(inventory, indent=2, sort_keys=True) + "\n", encoding="utf-8") + bindings = root / "bindings" + self.compiler.write_compilation(bindings, report) + return inventory_path, bindings, bindings / "compilation-report.json" + + def _prepare(self, root: Path, *, output: str = "release", apply: bool = False): + inventory, bindings, report = self._compiled_input(root) + return self.preparer.prepare_alpha_reader_release( + inventory_path=inventory, + bindings_dir=bindings, + report_path=report, + output_dir=root / output, + source_revision=SOURCE_REVISION, + image_reference="qdl-v2-python:2.0.0-dev-ccccccc", + image_id=IMAGE_ID, + rollback_images=ROLLBACK_IMAGES, + apply=apply, + ) + + def test_dry_run_is_secret_free_and_does_not_write(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + result = self._prepare(root, apply=False) + self.assertEqual(result["status"], "DRY_RUN") + self.assertFalse((root / "release").exists()) + self.assertFalse(result["secret_values_recorded"]) + self.assertNotIn("password", json.dumps(result, sort_keys=True).lower()) + + def test_apply_is_atomic_deterministic_and_has_only_reader_override(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + first = self._prepare(root, output="release-one", apply=True) + second = self._prepare(root, output="release-two", apply=True) + self.assertEqual(first["status"], "APPLIED") + first_dir = root / "release-one" + second_dir = root / "release-two" + self.assertEqual( + sorted(path.relative_to(first_dir).as_posix() for path in first_dir.rglob("*") if path.is_file()), + sorted(path.relative_to(second_dir).as_posix() for path in second_dir.rglob("*") if path.is_file()), + ) + for relative in ("inventory.json", "compilation-report.json", "reader-image.override.yml", "reader-rollback.override.yml", "release-manifest.json"): + self.assertEqual((first_dir / relative).read_bytes(), (second_dir / relative).read_bytes()) + manifest = json.loads((first_dir / "release-manifest.json").read_text(encoding="utf-8")) + self.assertEqual(manifest["schema"], "qdl.v2.alpha-reader-release.v2") + self.assertEqual(manifest["image_id"], IMAGE_ID) + self.assertEqual(manifest["image_selector"], "qdl-v2-python:2.0.0-dev-ccccccc@" + IMAGE_ID) + self.assertEqual(manifest["rollback_images"], ROLLBACK_IMAGES) + self.assertEqual(manifest["services"], list(self.preparer.READER_SERVICES)) + self.assertFalse(manifest["secret_values_recorded"]) + override = (first_dir / "reader-image.override.yml").read_text(encoding="utf-8") + self.assertEqual(override.count("image: qdl-v2-python:2.0.0-dev-ccccccc@" + IMAGE_ID), 4) + parsed_override = yaml.safe_load(override) + self.assertEqual(set(parsed_override), {"services"}) + self.assertEqual(set(parsed_override["services"]), set(self.preparer.READER_SERVICES)) + self.assertTrue( + all( + parsed_override["services"][service]["image"] + == "qdl-v2-python:2.0.0-dev-ccccccc@" + IMAGE_ID + for service in self.preparer.READER_SERVICES + ) + ) + parsed_rollback = yaml.safe_load( + (first_dir / "reader-rollback.override.yml").read_text(encoding="utf-8") + ) + self.assertEqual(set(parsed_rollback["services"]), set(self.preparer.READER_SERVICES)) + self.assertEqual( + (first_dir / "reader-rollback.override.yml").read_text(encoding="utf-8").count( + "image: qdl-v2-python:2.0.0-rollback-query-bbbbbbb@" + QUERY_ROLLBACK_IMAGE_ID + ), 2, + ) + self.assertEqual( + (first_dir / "reader-rollback.override.yml").read_text(encoding="utf-8").count( + "image: qdl-v2-python:2.0.0-rollback-stream-ccccccc@" + STREAM_ROLLBACK_IMAGE_ID + ), 2, + ) + self.assertNotIn("runtime", override.lower()) + self.assertNotIn("password", (first_dir / "release-manifest.json").read_text(encoding="utf-8").lower()) + binding_files = sorted((first_dir / "bindings").glob("*.binding.json")) + self.assertTrue(binding_files) + self.assertTrue( + all(stat.S_IMODE(path.stat().st_mode) == 0o444 for path in binding_files) + ) + self.assertEqual( + stat.S_IMODE((first_dir / "release-manifest.json").stat().st_mode), + 0o640, + ) + + def test_rejects_tampered_binding_and_existing_output(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + inventory, bindings, report = self._compiled_input(root) + binding = next(bindings.glob("*.binding.json")) + payload = json.loads(binding.read_text(encoding="utf-8")) + payload["consumer_id"] = "tampered.consumer" + binding.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + with self.assertRaisesRegex(self.preparer.ReleasePreparationError, "invalid|canonical|differ"): + self.preparer.prepare_alpha_reader_release( + inventory_path=inventory, + bindings_dir=bindings, + report_path=report, + output_dir=root / "release", + source_revision=SOURCE_REVISION, + image_reference="qdl-v2-python:2.0.0-dev-ccccccc", + image_id=IMAGE_ID, + rollback_images=ROLLBACK_IMAGES, + apply=False, + ) + self._prepare(root / "fresh", apply=True) + with self.assertRaises(FileExistsError): + self._prepare(root / "fresh", apply=True) + + def test_rejects_invalid_release_coordinates(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + inventory, bindings, report = self._compiled_input(root) + with self.assertRaisesRegex(self.preparer.ReleasePreparationError, "canonical reader tag"): + self.preparer.prepare_alpha_reader_release( + inventory_path=inventory, + bindings_dir=bindings, + report_path=report, + output_dir=root / "release", + source_revision=SOURCE_REVISION, + image_reference="not-an-image", + image_id=IMAGE_ID, + rollback_images=ROLLBACK_IMAGES, + apply=False, + ) + same_as_candidate = deepcopy(ROLLBACK_IMAGES) + same_as_candidate["query_v2_1"]["image_id"] = IMAGE_ID + with self.assertRaisesRegex(self.preparer.ReleasePreparationError, "must differ"): + self.preparer.prepare_alpha_reader_release( + inventory_path=inventory, + bindings_dir=bindings, + report_path=report, + output_dir=root / "release-two", + source_revision=SOURCE_REVISION, + image_reference="qdl-v2-python:2.0.0-dev-ccccccc", + image_id=IMAGE_ID, + rollback_images=same_as_candidate, + apply=False, + ) + incomplete = deepcopy(ROLLBACK_IMAGES) + incomplete.pop("stream_v2_passive") + with self.assertRaisesRegex(self.preparer.ReleasePreparationError, "cover exactly"): + self.preparer.prepare_alpha_reader_release( + inventory_path=inventory, + bindings_dir=bindings, + report_path=report, + output_dir=root / "release-three", + source_revision=SOURCE_REVISION, + image_reference="qdl-v2-python:2.0.0-dev-ccccccc", + image_id=IMAGE_ID, + rollback_images=incomplete, + apply=False, + ) + malformed = deepcopy(ROLLBACK_IMAGES) + malformed["stream_v2_active"]["image_reference"] = "not-an-image" + with self.assertRaisesRegex(self.preparer.ReleasePreparationError, "rollback image reference"): + self.preparer.prepare_alpha_reader_release( + inventory_path=inventory, + bindings_dir=bindings, + report_path=report, + output_dir=root / "release-four", + source_revision=SOURCE_REVISION, + image_reference="qdl-v2-python:2.0.0-dev-ccccccc", + image_id=IMAGE_ID, + rollback_images=malformed, + apply=False, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_alpha_workload_identity.py b/tests/test_alpha_workload_identity.py new file mode 100644 index 00000000..ae2fa01e --- /dev/null +++ b/tests/test_alpha_workload_identity.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path +import os +import stat +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts/materialize_alpha_v2_identity.py" + + +def _module(): + spec = importlib.util.spec_from_file_location("alpha_workload_identity", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class AlphaWorkloadIdentityTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.identity = _module() + + def _sources(self, root: Path) -> tuple[Path, Path]: + tls = root / "tls-source" + jwt = root / "jwt-source" + tls.mkdir() + jwt.mkdir() + for name in ("ca.crt", "client.crt", "client.key"): + (tls / name).write_text(f"{name}\n", encoding="utf-8") + for name in ("private.key", "public.pem"): + (jwt / name).write_text(f"{name}\n", encoding="utf-8") + (tls / "client.key").chmod(0o440) + (jwt / "private.key").chmod(0o440) + return tls, jwt + + def test_dry_run_is_secret_free_and_does_not_write(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + tls, jwt = self._sources(root) + state = root / "state" + state.mkdir() + result = self.identity.materialize_identity( + tls_source=tls, + jwt_source=jwt, + state_root=state, + workload_id="alpha.binance.paper", + workload_uid=1000, + workload_gid=1000, + apply=False, + ) + self.assertEqual(result["status"], "DRY_RUN") + self.assertFalse((state / "workload-identities").exists()) + self.assertFalse(result["secret_values_recorded"]) + self.assertNotIn("private.key", str(result)) + + @unittest.skipUnless(os.geteuid() == 0, "apply ownership test requires root") + def test_apply_is_atomic_and_private_for_declared_workload(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + tls, jwt = self._sources(root) + state = root / "state" + state.mkdir() + result = self.identity.materialize_identity( + tls_source=tls, + jwt_source=jwt, + state_root=state, + workload_id="alpha.okx.paper", + workload_uid=1000, + workload_gid=1000, + apply=True, + ) + target = Path(result["target"]) + self.assertTrue(target.is_dir()) + self.assertFalse(any(path.name.startswith(".") for path in target.parent.iterdir())) + self.assertEqual(stat.S_IMODE((target / "tls" / "client.key").stat().st_mode), 0o400) + self.assertEqual(stat.S_IMODE((target / "jwt" / "private.key").stat().st_mode), 0o400) + self.assertEqual(stat.S_IMODE((target / "tls" / "ca.crt").stat().st_mode), 0o444) + self.assertEqual((target / "tls" / "client.key").stat().st_uid, 1000) + self.assertEqual((target / "jwt" / "private.key").stat().st_gid, 1000) + self.assertEqual(stat.S_IMODE((target / "tls").stat().st_mode), 0o500) + self.assertEqual(stat.S_IMODE((target / "jwt").stat().st_mode), 0o500) + self.assertEqual(stat.S_IMODE((tls / "client.key").stat().st_mode), 0o440) + + def test_rejects_symlink_or_unexpected_source_file(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + tls, jwt = self._sources(root) + state = root / "state" + state.mkdir() + (tls / "extra.pem").write_text("extra\n", encoding="utf-8") + with self.assertRaisesRegex(self.identity.IdentityMaterializationError, "files differ"): + self.identity.materialize_identity( + tls_source=tls, + jwt_source=jwt, + state_root=state, + workload_id="alpha.binance.paper", + workload_uid=1000, + workload_gid=1000, + apply=False, + ) + (tls / "extra.pem").unlink() + (jwt / "link").symlink_to(jwt / "public.pem") + with self.assertRaisesRegex(self.identity.IdentityMaterializationError, "non-regular"): + self.identity.materialize_identity( + tls_source=tls, + jwt_source=jwt, + state_root=state, + workload_id="alpha.binance.paper", + workload_uid=1000, + workload_gid=1000, + apply=False, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_c419_fast_final_bar_delivery.py b/tests/test_c419_fast_final_bar_delivery.py index d1429e6a..196e538f 100644 --- a/tests/test_c419_fast_final_bar_delivery.py +++ b/tests/test_c419_fast_final_bar_delivery.py @@ -60,6 +60,7 @@ def _source(binding_id: str, *, interval: str = "1m", venue: str = "BINANCE"): def _edge(*pairs, clock, max_workers: int = 32) -> StableBinanceBarEdge: edge = object.__new__(StableBinanceBarEdge) + edge.repair_only = False edge.bindings = tuple(pair for pair in pairs if pair[1].runtime == "BINANCE") edge.okx_bindings = tuple(pair for pair in pairs if pair[1].runtime == "OKX") edge._rest_fallback_active = True diff --git a/tests/test_catalog_demand_consistency.py b/tests/test_catalog_demand_consistency.py index babbf163..866a8351 100644 --- a/tests/test_catalog_demand_consistency.py +++ b/tests/test_catalog_demand_consistency.py @@ -167,9 +167,18 @@ def test_non_minute_bar_freshness_covers_its_interval(self): ) checked.append((manifest.consumer_id, requirement.interval)) self.assertEqual( - checked, - [("alpha.binance.paper.stable", "15m")] * 5 - + [("alpha.okx.paper.stable", "1h")] * 5, + sorted(checked), + sorted( + [ + (consumer_id, interval) + for consumer_id in ( + "alpha.binance.paper.stable", + "alpha.okx.paper.stable", + ) + for interval in ("15m", "1h") + for _ in range(5) + ] + ), ) def test_a_requirement_no_source_can_answer_still_fails(self): @@ -179,10 +188,12 @@ def test_a_requirement_no_source_can_answer_still_fails(self): manifest = ConsumerManifestLoader.load( CONSUMER_DIR / "alpha-binance-paper.yaml" ) - served = next( - item for item in manifest.requirements - if item.recovery is RecoveryPolicy.FRESH_SNAPSHOT + materialized_bar = next( + item + for item in manifest.requirements + if item.feed is FeedType.BAR ) + served = replace(materialized_bar, recovery=RecoveryPolicy.FRESH_SNAPSHOT) self.assertTrue(pass_through_eligible(self.catalog, served)) # Asking for replay continuity: only a binding can promise that. diff --git a/tests/test_converge_v2_primary_runtime.py b/tests/test_converge_v2_primary_runtime.py new file mode 100644 index 00000000..b2605a80 --- /dev/null +++ b/tests/test_converge_v2_primary_runtime.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_deployment import ( + StableAcquisitionPlan, + stable_authority_record, + write_stable_runtime_bundle, +) +from scripts.converge_v2_primary_runtime import ( + CORE_FILES, + INGESTOR_FILES, + RUNTIME_FILES, + _FIVE_LIQUID_BOOK_IDS_BY_INGESTOR, + _FIVE_LIQUID_PERPETUAL_BOOK_IDS, + converge, +) + + +ROOT = Path(__file__).resolve().parents[1] +CATALOG = ROOT / "config/v2/stable-source-bindings.yaml" +ACQUISITION = ROOT / "config/v2/stable-acquisition-bindings.yaml" +RUST_IMAGE = "sha256:" + "a" * 64 + + +class PrimaryRuntimeConvergenceTests(unittest.TestCase): + def _authority(self) -> dict[str, object]: + return stable_authority_record( + rust_image_digest=RUST_IMAGE, + capability_manifest=ROOT / "config/v2/stable-capabilities.yaml", + contract=ROOT / "contracts/proto/qdl/marketdata/v2/market_data.proto", + partition_plan=ACQUISITION.read_bytes(), + effective_at_ns=1, + mode="RUST_PRIMARY", + revision=1, + slice_id="qdl-v2-primary-runtime-convergence-test", + approved_by="test", + ) + + def _legacy_runtime(self, root: Path) -> Path: + runtime = root / "runtime" + catalog = StableSourceCatalog.load(CATALOG) + acquisition = StableAcquisitionPlan.load(ACQUISITION, catalog=catalog) + write_stable_runtime_bundle( + runtime, + catalog=catalog, + acquisition=acquisition, + authority=self._authority(), + ) + for name in CORE_FILES: + path = runtime / name + payload = json.loads(path.read_text(encoding="utf-8")) + retained = [ + item + for item in payload["core"]["bindings"] + if item["source_id"].endswith("stable-001") + or item.get("native_symbol") in {"FPT", "VN30F1M"} + ] + self.assertTrue(retained) + for item in retained: + item["instrument_catalog_revision"] = 3 + item["instrument_revision"] = 1 + payload["core"]["bindings"] = retained + payload["core"]["dedup_capacity"] = 1_000_000 + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + path.chmod(0o640) + for name in INGESTOR_FILES: + path = runtime / name + payload = json.loads(path.read_text(encoding="utf-8")) + retained = [ + item + for item in payload["bindings"] + if item.get("native_symbol") in {"BTCUSDT", "ETHUSDT", "BTC-USDT-SWAP", "ETH-USDT-SWAP"} + and item.get("feed") in {"BAR", "TRADE", "QUOTE"} + ] + self.assertTrue(retained) + for item in retained: + item["instrument_catalog_revision"] = 3 + item["instrument_revision"] = 1 + payload["bindings"] = retained + payload["config_revision"] = 9 + payload.pop("session_liveness_dir", None) + payload.pop("session_liveness_write_interval_ms", None) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + path.chmod(0o640) + return runtime + + def test_dry_run_proves_partial_runtime_converges_without_mutation(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + runtime = self._legacy_runtime(root) + before = {name: (runtime / name).read_bytes() for name in RUNTIME_FILES} + result = converge( + runtime_dir=runtime, + output_dir=None, + apply=False, + state_root=root / "state", + ) + self.assertEqual(result["status"], "DRY_RUN") + self.assertEqual(result["production_mutations"], 0) + self.assertTrue(result["authority_bytes_preserved"]) + for name in CORE_FILES: + item = result["files"][name] + self.assertGreater(item["after_binding_count"], item["before_binding_count"]) + self.assertEqual( + set(item["added_five_liquid_book_source_ids"]), + _FIVE_LIQUID_PERPETUAL_BOOK_IDS, + ) + self.assertEqual(item["dedup_capacity"], {"before": 1_000_000, "after": 100_000}) + for name in INGESTOR_FILES: + item = result["files"][name] + self.assertGreater(item["after_binding_count"], item["before_binding_count"]) + self.assertTrue( + _FIVE_LIQUID_BOOK_IDS_BY_INGESTOR[name] + <= set(item["added_book_subscription_ids"]) + ) + self.assertEqual(item["session_liveness_write_interval_ms"], 1_000) + self.assertEqual({name: (runtime / name).read_bytes() for name in RUNTIME_FILES}, before) + + def test_apply_writes_all_five_backups_and_preserves_authority(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + runtime = self._legacy_runtime(root) + authority_before = (runtime / "authority.json").read_bytes() + before = {name: (runtime / name).read_bytes() for name in RUNTIME_FILES} + modes = {name: (runtime / name).stat().st_mode & 0o777 for name in RUNTIME_FILES} + result = converge( + runtime_dir=runtime, + output_dir=root / "state" / "primary-runtime-convergence", + apply=True, + state_root=root / "state", + ) + self.assertEqual(result["production_mutations"], 5) + self.assertEqual((runtime / "authority.json").read_bytes(), authority_before) + for name in RUNTIME_FILES: + backup = root / "state" / "primary-runtime-convergence" / "rollback" / name + self.assertEqual(backup.read_bytes(), before[name]) + self.assertEqual(backup.stat().st_mode & 0o777, modes[name]) + self.assertEqual((runtime / name).stat().st_mode & 0o777, modes[name]) + self.assertTrue((root / "state" / "primary-runtime-convergence" / "receipt.json").is_file()) + + def test_rejects_retained_binding_semantic_drift(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + runtime = self._legacy_runtime(root) + path = runtime / "ingestor-binance-usdm.json" + payload = json.loads(path.read_text(encoding="utf-8")) + payload["bindings"][0]["native_channel"] = "invalid" + path.write_text(json.dumps(payload), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "semantic drift"): + converge( + runtime_dir=runtime, + output_dir=None, + apply=False, + state_root=root / "state", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_execution_l2_materialization.py b/tests/test_execution_l2_materialization.py new file mode 100644 index 00000000..d7396e6b --- /dev/null +++ b/tests/test_execution_l2_materialization.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import copy +import tempfile +import unittest +from pathlib import Path + +import yaml + +from qdl.runtime.execution_l2 import execution_l2_materialization_plan +from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_deployment import StableAcquisitionPlan + + +ROOT = Path(__file__).resolve().parents[1] +CATALOG = ROOT / "config/v2/stable-source-bindings.yaml" +ACQUISITION = ROOT / "config/v2/stable-acquisition-bindings.yaml" +DEMAND = ROOT / "config/v2/stable-crypto-demand.yaml" +EXPECTED_SOURCE_IDS = frozenset({ + "binance-usdm-bnbusdt-book-primary-v2", + "binance-usdm-btcusdt-book-primary-v2", + "binance-usdm-dogeusdt-book-primary-v2", + "binance-usdm-ethusdt-book-primary-v2", + "binance-usdm-solusdt-book-primary-v2", + "okx-swap-bnb-usdt-swap-book-primary-v2", + "okx-swap-btc-usdt-swap-book-primary-v2", + "okx-swap-doge-usdt-swap-book-primary-v2", + "okx-swap-eth-usdt-swap-book-primary-v2", + "okx-swap-sol-usdt-swap-book-primary-v2", +}) + + +class ExecutionL2MaterializationTests(unittest.TestCase): + def setUp(self) -> None: + self.catalog = StableSourceCatalog.load(CATALOG) + self.acquisition = StableAcquisitionPlan.load(ACQUISITION, catalog=self.catalog) + + def test_manifest_derives_all_current_execution_books_without_symbol_policy_code(self) -> None: + plan = execution_l2_materialization_plan( + demand_path=DEMAND, + catalog=self.catalog, + acquisition=self.acquisition, + ) + self.assertEqual(set(plan.source_ids), EXPECTED_SOURCE_IDS) + self.assertEqual(len(plan.binding_ids), 20) + self.assertEqual(plan.materialized_snapshot_interval_ms, 1_000) + + catalog_by_binding = { + binding.binding_id: binding for binding in self.catalog.bindings + } + bindings = { + catalog_by_binding[binding.binding_id].source_id: binding + for binding in self.acquisition.bindings + if binding.binding_id in catalog_by_binding + and catalog_by_binding[binding.binding_id].source_id in plan.source_ids + } + self.assertEqual(set(bindings), EXPECTED_SOURCE_IDS) + self.assertTrue(all( + binding.mode == "RUST_NATIVE" + and binding.sequence_policy == "CONTIGUOUS" + and binding.l2 is not None + and binding.l2.depth_per_side == 100 + and binding.l2.snapshot_refresh_seconds == 30 + and binding.l2.materialized_snapshot_interval_ms == 1_000 + for binding in bindings.values() + )) + + def test_fails_closed_when_an_execution_book_pair_is_incomplete(self) -> None: + raw = yaml.safe_load(DEMAND.read_text(encoding="utf-8")) + consumer = next( + item for item in raw["consumers"] + if item["consumer_grade"] == "EXECUTION" + ) + delta_index = next( + index for index, item in enumerate(consumer["requirements"]) + if item["feed"] == "BOOK_DELTA" + ) + consumer["requirements"].pop(delta_index) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "demand.yaml" + path.write_text(yaml.safe_dump(raw, sort_keys=False), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "complete snapshot/delta pair"): + execution_l2_materialization_plan( + demand_path=path, + catalog=self.catalog, + acquisition=self.acquisition, + ) + + def test_fails_closed_when_one_execution_book_loses_hot_materialization(self) -> None: + raw = yaml.safe_load(ACQUISITION.read_text(encoding="utf-8")) + changed = copy.deepcopy(raw) + binding = next( + item for item in changed["bindings"] + if item["binding_id"] == "okx-swap-sol-usdt-swap-book_snapshot" + ) + binding["l2"].pop("materialized_snapshot_interval_ms") + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "acquisition.yaml" + path.write_text(yaml.safe_dump(changed, sort_keys=False), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "complete equivalent book pair"): + StableAcquisitionPlan.load(path, catalog=self.catalog) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fund_phase2_transport.py b/tests/test_fund_phase2_transport.py index 3bfe5ab5..c0b46e58 100644 --- a/tests/test_fund_phase2_transport.py +++ b/tests/test_fund_phase2_transport.py @@ -79,6 +79,7 @@ def spool(self, **overrides) -> SQLiteDurableSpool: replay_retention_seconds=10, maintenance_interval_seconds=1, max_partition_records=overrides.get("max_partition_records", 0), + retain_partition_windows=overrides.get("retain_partition_windows", False), ) return SQLiteDurableSpool(config, clock_ns=self.clock) @@ -145,6 +146,56 @@ def test_tail_returns_newest_window_without_changing_replay_order(self): [row.event.payload for row in latest], [b"event-4", b"event-5"] ) + def test_tail_allows_only_configured_internal_partition_headroom(self): + with self.spool(max_partition_records=10_064) as spool: + self.assertEqual( + spool.read_tail( + stream=event(1).stream, + partition_key=event(1).partition_key, + limit=10_064, + ), + [], + ) + with self.spool() as spool: + with self.assertRaisesRegex(ValueError, "between 1 and 10000"): + spool.read_tail( + stream=event(1).stream, + partition_key=event(1).partition_key, + limit=10_001, + ) + + def test_retained_window_survives_age_and_restart_but_remains_count_bounded(self): + with self.spool(max_partition_records=3, retain_partition_windows=True) as spool: + first = spool.append(event(1)) + spool.append(event(2)) + self.clock.now_ns += 86_400_000_000_000 + spool.append(event(3)) + self.assertEqual(spool.stats().records, 3) + self.assertTrue(spool.append(event(1)).duplicate) + self.assertEqual(spool.trim_consumed(), 0) + with self.spool(max_partition_records=3, retain_partition_windows=True) as spool: + self.assertEqual(spool.stats().records, 3) + spool.append(event(4)) + spool.append(event(5)) + self.assertEqual(spool.stats().records, 3) + with self.assertRaises(CursorExpired): + spool.read(stream=event(1).stream, partition_key=event(1).partition_key, + after=first.cursor) + self.assertTrue(spool.integrity_check()) + + def test_default_partition_window_still_expires_by_age(self): + with self.spool(max_partition_records=3) as spool: + spool.append(event(1)) + self.clock.now_ns += 11_000_000_000 + spool.append(event(2)) + self.assertIsNone(spool.find_event(stream=event(1).stream, event_id=event(1).event_id)) + + def test_retained_partition_window_requires_finite_bound(self): + for value, bound in ((True, 0), ("true", 3)): + with self.subTest(value=value, bound=bound), self.assertRaises(ValueError): + SpoolConfig(path=self.path, retain_partition_windows=value, + max_partition_records=bound) + def test_partition_window_is_bounded_and_old_cursor_expires(self): with self.spool(max_records=10, max_partition_records=3) as spool: spool.append_many([event(index) for index in range(1, 6)]) diff --git a/tests/test_fund_phase5_contracts.py b/tests/test_fund_phase5_contracts.py index 02ec84a7..65392624 100644 --- a/tests/test_fund_phase5_contracts.py +++ b/tests/test_fund_phase5_contracts.py @@ -4,6 +4,9 @@ import unittest from pathlib import Path +from pydantic import ValidationError + +from qdl.api_v2.models import MAX_REQUIREMENT_FRESHNESS_MS, RequirementModel from scripts.generate_phase5_openapi import build_openapi @@ -11,6 +14,45 @@ class Phase5OpenApiContractTests(unittest.TestCase): + def test_final_bar_freshness_bound_is_uniform_and_fail_closed(self): + payload = { + "instrument_uid": "BINANCE.USDM.PERPETUAL.BTC-USDT", + "feed": "BAR", + "consumer_grade": "ALPHA", + "source_policy_id": "alpha_crypto_primary_v2", + "interval": "1w", + "max_freshness_ms": MAX_REQUIREMENT_FRESHNESS_MS, + } + self.assertEqual( + RequirementModel.model_validate(payload).max_freshness_ms, + MAX_REQUIREMENT_FRESHNESS_MS, + ) + with self.assertRaises(ValidationError): + RequirementModel.model_validate( + {**payload, "max_freshness_ms": MAX_REQUIREMENT_FRESHNESS_MS + 1} + ) + + openapi = build_openapi() + requirement_schema = openapi["components"]["schemas"]["RequirementModel"] + self.assertEqual( + requirement_schema["properties"]["max_freshness_ms"]["anyOf"][0]["maximum"], + float(MAX_REQUIREMENT_FRESHNESS_MS), + ) + for route in ( + "/v2/market-data/{instrument_uid}/snapshot", + "/v2/market-data/{instrument_uid}/warmup", + "/v2/market-data/{instrument_uid}/history", + "/v2/feeds/{instrument_uid}/status", + ): + parameters = openapi["paths"][route]["get"]["parameters"] + freshness = next( + item for item in parameters if item["name"] == "max_freshness_ms" + ) + self.assertEqual( + freshness["schema"]["anyOf"][0]["maximum"], + MAX_REQUIREMENT_FRESHNESS_MS, + ) + def test_v2_openapi_matches_frozen_snapshot_and_has_typed_public_responses(self): expected = json.loads( (ROOT / "contracts/v2/openapi.snapshot.json").read_text(encoding="utf-8") diff --git a/tests/test_fund_phase83_release.py b/tests/test_fund_phase83_release.py index 78e44ab1..50e94465 100644 --- a/tests/test_fund_phase83_release.py +++ b/tests/test_fund_phase83_release.py @@ -2,7 +2,10 @@ import json import pathlib -import tomllib +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib import unittest import yaml diff --git a/tests/test_phase103_consumer_acceptance.py b/tests/test_phase103_consumer_acceptance.py index e47db24f..8c53d3e5 100644 --- a/tests/test_phase103_consumer_acceptance.py +++ b/tests/test_phase103_consumer_acceptance.py @@ -24,7 +24,9 @@ ) from qdl.runtime.stable_catalog import StableSourceCatalog from qdl.runtime.stable_deployment import StableAcquisitionPlan +from qdl.query.contracts import FeedType, RecoveryPolicy, StalePolicy from scripts.phase103_consumer_receipt_acceptance import ( + _c2_requirement, _query_product, _query_product_with_quality, _stream_event_timeout_seconds, @@ -62,18 +64,17 @@ def test_governed_manifests_cover_exact_crypto_products_and_vn_is_explicitly_def {item.consumer_id for item in scope.products}, PHASE103_CONSUMER_IDS, ) - self.assertEqual(len(scope.products), 45) + self.assertEqual(len(scope.products), 110) self.assertEqual( sum(item.delivery is DeliveryClass.DURABLE for item in scope.products), - 40, + 110, ) pass_through = [ item for item in scope.products if item.delivery is DeliveryClass.PROVIDER_PASS_THROUGH ] - self.assertEqual(len(pass_through), 5) - self.assertTrue(all(item.binding_id is not None for item in pass_through)) + self.assertEqual(pass_through, []) self.assertEqual( { item.binding_id @@ -92,7 +93,7 @@ def test_governed_manifests_cover_exact_crypto_products_and_vn_is_explicitly_def "binance-usdm-solusdt-bar-15m", }, ) - self.assertEqual(len(scope.excluded), 66) + self.assertEqual(len(scope.excluded), 76) excluded = next( item for item in scope.excluded if item.reason == "VENUE_NOT_IN_PHASE103_CRYPTO_SCOPE" @@ -104,7 +105,7 @@ def test_governed_manifests_cover_exact_crypto_products_and_vn_is_explicitly_def item for item in scope.excluded if item.reason == "LATER_PHASE_PRODUCT" ] - self.assertEqual(len(later_phase), 65) + self.assertEqual(len(later_phase), 75) self.assertEqual( {item.consumer_id for item in later_phase}, {"trading-system.paper.stable", "alpha.binance.paper.stable"}, @@ -206,6 +207,39 @@ def _product( ) ) + def _pass_through_bar(self): + """Exercise the legal provider history path without weakening the manifest.""" + durable = self._product( + feed="BAR", + consumer_id="alpha.binance.paper.stable", + interval="15m", + ) + return replace( + durable, + delivery=DeliveryClass.PROVIDER_PASS_THROUGH, + requirement=replace( + durable.requirement, + recovery=RecoveryPolicy.FRESH_SNAPSHOT, + ), + ) + + def _quiet_book_delta_product(self): + """Build one governed quiet-delta receipt without weakening price rules.""" + trade = self._product(feed="TRADE") + return replace( + trade, + feed=FeedType.BOOK_DELTA, + requirement=replace( + trade.requirement, + feed=FeedType.BOOK_DELTA, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + event_recency_policy=StalePolicy.OBSERVE, + max_session_liveness_ms=45_000, + require_final_bars=False, + ), + ) + def _view( self, product, @@ -238,6 +272,22 @@ def _view( "quantity_unit": "BASE_ASSET", "level": 1, } + elif product.feed.value == "BOOK_DELTA": + payload = { + "feed": "BOOK_DELTA", + "native_sequence_start": "101", + "native_sequence_end": "102", + "snapshot_sequence": "100", + "updates": [{ + "side": "BID", + "price": d("10.1"), + "quantity": d("1"), + "quantity_unit": "BASE_ASSET", + }], + "reset": False, + "book_generation": 1, + "sequence_verified": True, + } else: payload = { "feed": "BAR", @@ -366,18 +416,103 @@ def test_quiet_connected_trade_is_accepted_for_no_order_observation_only(self): with self.assertRaisesRegex(ValueError, "provider session"): validate_product_view(product, disconnected) + def test_state_replay_keeps_identity_and_gap_checks_but_defers_old_session_quality(self): + product = self._quiet_book_delta_product() + stale = self._view( + product, + freshness_ms=product.requirement.max_freshness_ms + 1, + execution_eligible=False, + ).model_copy( + update={ + "quality": self._view( + product, + freshness_ms=product.requirement.max_freshness_ms + 1, + execution_eligible=False, + ).quality.model_copy( + update={ + "state": "STALE", + "event_recency_state": "STALE", + "provider_session_state": "STALE", + "provider_session_liveness_ms": 45_001, + } + ) + } + ) + with self.assertRaisesRegex(ValueError, "provider session"): + validate_product_view(product, stale, require_current_quality=False) + validate_product_view( + product, + stale, + require_current_quality=False, + state_replay=True, + ) + with self.assertRaisesRegex(ValueError, "cannot claim current"): + validate_product_view(product, stale, state_replay=True) + + def test_quiet_connected_book_delta_is_observable_but_never_price_eligible(self): + product = self._quiet_book_delta_product() + quiet = self._view( + product, + freshness_ms=product.requirement.max_freshness_ms + 1, + execution_eligible=False, + ) + quiet = quiet.model_copy( + update={ + "quality": quiet.quality.model_copy( + update={ + "event_recency_state": "STALE", + "provider_session_state": "LIVE", + "provider_session_liveness_ms": 1, + } + ) + } + ) + validate_product_view(product, quiet) + + blocked = replace( + product, + requirement=replace(product.requirement, event_recency_policy=None), + ) + with self.assertRaisesRegex(ValueError, "freshness"): + validate_product_view(blocked, quiet) + + disconnected = quiet.model_copy( + update={ + "quality": quiet.quality.model_copy( + update={"provider_session_state": "DISCONNECTED"} + ) + } + ) + with self.assertRaisesRegex(ValueError, "provider session"): + validate_product_view(product, disconnected) + + with self.assertRaisesRegex(ValueError, "gap"): + validate_product_view( + product, + quiet.model_copy( + update={"quality": quiet.quality.model_copy(update={"gap_open": True})} + ), + ) + + with self.assertRaisesRegex(ValueError, "verified"): + validate_product_view( + product, + quiet.model_copy( + update={ + "payload": quiet.payload.model_copy( + update={"sequence_verified": False} + ) + } + ), + ) + def test_typed_views_enforce_durable_and_pass_through_domain_semantics(self): for feed in ("TRADE", "QUOTE", "BAR"): with self.subTest(feed=feed): product = self._product(feed=feed) validate_product_view(product, self._view(product)) - pass_through = self._product( - feed="BAR", - delivery=DeliveryClass.PROVIDER_PASS_THROUGH, - consumer_id="alpha.binance.paper.stable", - interval="15m", - ) + pass_through = self._pass_through_bar() validate_product_view(pass_through, self._view(pass_through)) baseline = self._view(pass_through) durable_upgrade = baseline.model_copy( @@ -495,6 +630,12 @@ def window(opens, *, closes=None): rollover["secondary_content_sha256"], ) + two_row_rollover = validate_final_bar_warmup_windows(primary[-2:], secondary[-2:]) + self.assertEqual(two_row_rollover["common_row_count"], 1) + self.assertEqual(two_row_rollover["comparison"], "SINGLE_FINAL_BAR_ROLLOVER") + with self.assertRaisesRegex(ValueError, "no immutable common window"): + validate_final_bar_warmup_windows(primary[-1:], secondary[-1:]) + def test_final_bar_warmup_window_rejects_immutable_or_multi_row_divergence(self): bar = self._product(feed="BAR") minute = 60_000_000_000 @@ -561,8 +702,8 @@ async def warmup(self, requirement): _query_product(bar, primary=primary, secondary=secondary) ) self.assertNotEqual(primary_hash, secondary_hash) - self.assertEqual(primary.requirement, sdk_requirement(bar)) - self.assertEqual(secondary.requirement, sdk_requirement(bar)) + self.assertEqual(primary.requirement, _c2_requirement(sdk_requirement(bar))) + self.assertEqual(secondary.requirement, _c2_requirement(sdk_requirement(bar))) quality_primary = WarmupClient(primary_rows) quality_secondary = WarmupClient(secondary_rows) @@ -577,12 +718,7 @@ async def warmup(self, requirement): self.assertEqual(result[5]["common_row_count"], 2) def test_provider_pass_through_bar_keeps_exact_replica_comparison(self): - bar = self._product( - feed="BAR", - delivery=DeliveryClass.PROVIDER_PASS_THROUGH, - consumer_id="alpha.binance.paper.stable", - interval="15m", - ) + bar = self._pass_through_bar() minute = 60_000_000_000 class WarmupClient: @@ -665,7 +801,7 @@ async def warmup(self, requirement): _query_product(bar, primary=primary, secondary=secondary) ) self.assertEqual(primary_hash, secondary_hash) - self.assertEqual(primary.requirement, sdk_requirement(bar)) + self.assertEqual(primary.requirement, _c2_requirement(sdk_requirement(bar))) def test_bar_stream_wait_covers_one_close_but_stays_sla_bounded(self): bar = self._product( @@ -678,12 +814,7 @@ def test_bar_stream_wait_covers_one_close_but_stays_sla_bounded(self): _stream_event_timeout_seconds(bar, 15.0) * 1_000, bar.requirement.max_freshness_ms, ) - alpha_bar = self._product( - feed="BAR", - delivery=DeliveryClass.PROVIDER_PASS_THROUGH, - consumer_id="alpha.binance.paper.stable", - interval="15m", - ) + alpha_bar = self._pass_through_bar() self.assertEqual(_stream_event_timeout_seconds(alpha_bar, 15.0), 915.0) trade = self._product(feed="TRADE") self.assertEqual(_stream_event_timeout_seconds(trade, 15.0), 15.0) diff --git a/tests/test_phase103_consumer_receipt_harness.py b/tests/test_phase103_consumer_receipt_harness.py index 78b7033d..e59a4789 100644 --- a/tests/test_phase103_consumer_receipt_harness.py +++ b/tests/test_phase103_consumer_receipt_harness.py @@ -7,17 +7,22 @@ import tempfile import time import unittest -from unittest.mock import ANY, call, patch +from unittest.mock import ANY, AsyncMock, call, patch from scripts.phase103_consumer_receipt_acceptance import ( + C2StatusEvidenceError, + _c2_requirement, + _certify_product, _cursor_directory, _historical_bar_replay_requirement, + _replay_precedes_handoff, _strict_snapshot_for_c2, _strict_warmup_then_stream_for_c2, _stream_handoff_mode, _stream_resume, _uses_historical_bar_replay, _validated_packet, + compact_feed_status, parser, ) from scripts.phase103_prepare_shared_primary_packet import ( @@ -46,13 +51,61 @@ StalePolicy, StreamEvent, ) +from qdl_sdk.models import SnapshotResponse from qdl_sdk.errors import ContinuityError, DataLayerError +from qdl.adapters.intervals import canonical_interval_ms, provider_bar_calendar_anchor_ms +from qdl.runtime.stable_bar_edge import durable_bar_history_capacity_rows ROOT = Path(__file__).resolve().parents[1] class Phase103ConsumerReceiptHarnessTests(unittest.TestCase): + def test_reconnect_replay_before_new_handoff_is_state_only(self): + self.assertTrue(_replay_precedes_handoff(logical_offset=40, watermark_offset=40)) + self.assertTrue(_replay_precedes_handoff(logical_offset=39, watermark_offset=40)) + self.assertFalse(_replay_precedes_handoff(logical_offset=41, watermark_offset=40)) + with self.assertRaisesRegex(ValueError, "non-negative"): + _replay_precedes_handoff(logical_offset=-1, watermark_offset=40) + + def test_c2_bounds_bar_history_without_reducing_the_public_request(self): + requirement = DataRequirement( + instrument_uid="bar-uid", + feed=Feed.BAR, + consumer_grade=Grade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="12h", + warmup_limit=10_000, + max_freshness_ms=86_400_000, + ) + bounded = _c2_requirement(requirement) + self.assertEqual(requirement.warmup_limit, 10_000) + self.assertEqual(bounded.warmup_limit, 700) + + def test_c2_uses_the_shared_durable_capacity_for_calendar_bars(self): + requirement = DataRequirement( + instrument_uid="bar-uid", + feed=Feed.BAR, + consumer_grade=Grade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1w", + warmup_limit=10_000, + max_freshness_ms=604_800_000, + ) + bounded = _c2_requirement(requirement) + self.assertEqual(durable_bar_history_capacity_rows("1w"), 156) + self.assertEqual(bounded.warmup_limit, 156) + + def test_c2_keeps_non_bar_requirement_exact(self): + requirement = DataRequirement( + instrument_uid="trade-uid", + feed=Feed.TRADE, + consumer_grade=Grade.ALPHA, + source_policy_id="crypto_primary_v2", + max_freshness_ms=3_000, + ) + self.assertIs(_c2_requirement(requirement), requirement) + def test_parser_requires_the_sealed_handoff_coordinates(self): parsed = parser().parse_args( [ @@ -154,6 +207,56 @@ def test_runbook_uses_immutable_image_for_packet_preflight(self): self.assertIn("docker run --rm --entrypoint sha256sum", runbook) +class Phase103C2StreamOpeningTests(unittest.IsolatedAsyncioTestCase): + async def test_certify_product_forwards_the_bounded_stream_open_window(self): + product = SimpleNamespace() + primary = SimpleNamespace(close=AsyncMock()) + secondary = SimpleNamespace(close=AsyncMock()) + stream_resume = AsyncMock(return_value=(None, None, (), ())) + + with tempfile.TemporaryDirectory(prefix="qdl-c2-stream-open-window-") as raw: + with ( + patch( + "scripts.phase103_consumer_receipt_acceptance._receipt_client", + side_effect=(primary, secondary), + ), + patch( + "scripts.phase103_consumer_receipt_acceptance._query_product_with_quality", + new=AsyncMock(return_value=("a", "b", 1.0, 2.0, {}, None)), + ), + patch( + "scripts.phase103_consumer_receipt_acceptance._stream_resume", + new=stream_resume, + ), + patch( + "scripts.phase103_consumer_receipt_acceptance.compact_receipt_evidence", + return_value={}, + ), + patch( + "scripts.phase103_consumer_receipt_acceptance._stream_handoff_mode", + return_value="NOT_APPLICABLE", + ), + ): + await _certify_product( + product, + identity=SimpleNamespace(), + primary_url="https://query-primary", + secondary_url="https://query-secondary", + grpc_target="stream:8210", + state_dir=Path(raw), + timeout_seconds=1.0, + stream_open_timeout_seconds=5.0, + ) + + stream_resume.assert_awaited_once() + self.assertEqual( + stream_resume.await_args.kwargs["stream_open_timeout_seconds"], + 5.0, + ) + primary.close.assert_awaited_once() + secondary.close.assert_awaited_once() + + class Phase103HistoricalBarReplayTests(unittest.TestCase): INTERVAL_NS = 15 * 60 * 1_000_000_000 @@ -238,6 +341,27 @@ def test_historical_seed_fails_closed_for_execution_unaligned_or_unbounded_requi latest_open_time_ns=20 * self.INTERVAL_NS, ) + def test_historical_seed_uses_shared_provider_calendar_anchors(self): + for interval, venue in ( + ("1w", "BINANCE"), + ("1w", "OKX"), + ("3d", "BINANCE"), + ("3d", "OKX"), + ): + interval_ns = canonical_interval_ms(interval) * 1_000_000 + anchor_ns = provider_bar_calendar_anchor_ms( + interval, + provider=venue, + ) * 1_000_000 + latest_open_ns = anchor_ns + 20 * interval_ns + seed = _historical_bar_replay_requirement( + self._requirement(interval=interval), + latest_open_time_ns=latest_open_ns, + calendar_provider=venue, + ) + + self.assertEqual(seed.warmup.time_range.start_time_ns, latest_open_ns - 2 * interval_ns) + self.assertEqual(seed.warmup.time_range.end_time_ns, latest_open_ns - interval_ns) def test_historical_replay_applies_only_to_non_execution_durable_bars(self): self.assertTrue(_uses_historical_bar_replay(self._product())) self.assertFalse( @@ -251,16 +375,32 @@ class Phase103HistoricalBarReplayResumeTests(unittest.IsolatedAsyncioTestCase): INTERVAL_NS = 15 * 60 * 1_000_000_000 class _Session: - def __init__(self, *, warmup, items): + def __init__( + self, + *, + warmup, + items, + quiet: bool = False, + first_item_delay_seconds: float = 0.0, + ): self.warmup = warmup self._items = iter(items) + self._quiet = quiet + self._first_item_delay_seconds = first_item_delay_seconds + self._first_item_pending = True self.acknowledged = [] async def __anext__(self): + if self._first_item_pending: + self._first_item_pending = False + if self._first_item_delay_seconds: + await asyncio.sleep(self._first_item_delay_seconds) await asyncio.sleep(0) try: return next(self._items) except StopIteration as error: + if self._quiet: + await asyncio.Event().wait() raise StopAsyncIteration from error def acknowledge(self, event): @@ -277,11 +417,13 @@ async def __aexit__(self, exc_type, exc, traceback): return False class _Client: - def __init__(self, *, strict_warmup, session): + def __init__(self, *, strict_warmup, session, snapshots=()): self.strict_warmup = strict_warmup self.session = session + self._snapshots = iter(snapshots) self.warmup_calls = [] self.stream_calls = [] + self.snapshot_calls = [] self.closed = False async def warmup(self, requirement): @@ -292,6 +434,13 @@ def warmup_then_stream(self, requirement, *, resume_restored_state=False): self.stream_calls.append((requirement, resume_restored_state)) return Phase103HistoricalBarReplayResumeTests._SessionContext(self.session) + async def snapshot(self, requirement): + self.snapshot_calls.append(requirement) + try: + return next(self._snapshots) + except StopIteration as error: + raise AssertionError("unexpected strict historical BAR snapshot") from error + async def close(self): self.closed = True @@ -321,7 +470,7 @@ def _requirement(*, grade: Grade = Grade.ALPHA): stale_policy=StalePolicy.BLOCK, ) - async def test_non_execution_bar_replays_retained_offsets_across_replicas(self): + async def test_non_execution_bar_uses_current_handoff_boundary_across_replicas(self): product = self._product() requirement = self._requirement() strict_view = SimpleNamespace( @@ -340,9 +489,10 @@ async def test_non_execution_bar_replays_retained_offsets_across_replicas(self): first_client = self._Client(strict_warmup=strict_warmup, session=first_session) resumed_client = self._Client(strict_warmup=strict_warmup, session=resumed_session) projected = [] + current = SimpleNamespace(data=SimpleNamespace()) - def project(event, *, template, requirement): - projected.append((event.logical_offset, template, requirement)) + def project(event, *, template, requirement, **kwargs): + projected.append((event.logical_offset, template, requirement, kwargs)) return SimpleNamespace(logical_offset=event.logical_offset) with tempfile.TemporaryDirectory(prefix="qdl-c2-historical-bar-") as raw: @@ -351,6 +501,10 @@ def project(event, *, template, requirement): patch("scripts.phase103_consumer_receipt_acceptance._client", side_effect=(first_client, resumed_client)), patch("scripts.phase103_consumer_receipt_acceptance.market_data_view_from_stream", side_effect=project), patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view") as validate, + patch( + "scripts.phase103_consumer_receipt_acceptance._strict_snapshot_for_c2", + return_value=current, + ) as readback, ): result = await _stream_resume( product, @@ -363,44 +517,50 @@ def project(event, *, template, requirement): ) self.assertEqual(result, (39, 40, ("REPLAYING",), ())) - self.assertEqual(first_client.warmup_calls, [requirement]) + self.assertEqual(first_client.warmup_calls, []) self.assertTrue(first_client.closed) self.assertTrue(resumed_client.closed) self.assertEqual(first_session.acknowledged, [first]) self.assertEqual(resumed_session.acknowledged, [resumed]) - seed_requirement, first_resume = first_client.stream_calls[0] + stream_requirement, first_resume = first_client.stream_calls[0] resumed_requirement, resumed_flag = resumed_client.stream_calls[0] self.assertFalse(first_resume) self.assertTrue(resumed_flag) - self.assertIs(seed_requirement, resumed_requirement) - self.assertEqual(seed_requirement.warmup.time_range.start_time_ns, 18 * self.INTERVAL_NS) - self.assertEqual(seed_requirement.warmup.time_range.end_time_ns, 19 * self.INTERVAL_NS) - self.assertEqual([offset for offset, _template, _requirement in projected], [39, 40]) - self.assertTrue(all(item[2] is seed_requirement for item in projected)) - self.assertEqual(validate.call_args_list[0], call(product, strict_view)) + self.assertEqual(stream_requirement, requirement) + self.assertEqual(resumed_requirement, requirement) + self.assertEqual([item[0] for item in projected], [39, 40]) + self.assertTrue(all(item[2] == requirement for item in projected)) + self.assertEqual(projected[0][3], {}) + self.assertEqual(projected[1][3], {"replay_only": True}) + readback.assert_awaited_once() self.assertEqual( - validate.call_args_list[1:], + validate.call_args_list, [ - call(product, ANY, require_current_quality=False), - call(product, ANY, require_current_quality=False), + call(product, ANY, require_current_quality=True), + call(product, ANY, require_current_quality=False, state_replay=True), + call(product, current.data), ], ) - async def test_non_execution_bar_drains_bounded_replay_to_strict_watermark(self): + async def test_non_execution_bar_replay_revalidates_current_snapshot_after_large_backfill_tail(self): product = self._product() requirement = self._requirement() strict_view = SimpleNamespace( payload=SimpleNamespace(open_time_ns=20 * self.INTERVAL_NS) ) seed_view = SimpleNamespace(payload=SimpleNamespace(open_time_ns=18 * self.INTERVAL_NS)) - strict_warmup = SimpleNamespace(data=[strict_view], watermark_offset=42) - seed_warmup = SimpleNamespace(data=[seed_view], watermark_offset=38) - first = StreamEvent(39, "resume-39", object()) - resumed = tuple(StreamEvent(offset, f"resume-{offset}", object()) for offset in (40, 41, 42)) + strict_warmup = SimpleNamespace(data=[strict_view], watermark_offset=11_024) + seed_warmup = SimpleNamespace(data=[seed_view], watermark_offset=10_004) + first = StreamEvent(10_005, "resume-10005", object()) + resumed = tuple( + StreamEvent(offset, f"resume-{offset}", object()) + for offset in (10_006, 10_007, 10_008) + ) first_session = self._Session(warmup=seed_warmup, items=(first,)) resumed_session = self._Session(warmup=seed_warmup, items=resumed) first_client = self._Client(strict_warmup=strict_warmup, session=first_session) resumed_client = self._Client(strict_warmup=strict_warmup, session=resumed_session) + current = SimpleNamespace(data=SimpleNamespace()) with tempfile.TemporaryDirectory(prefix="qdl-c2-historical-drain-") as raw: with ( @@ -408,8 +568,568 @@ async def test_non_execution_bar_drains_bounded_replay_to_strict_watermark(self) patch("scripts.phase103_consumer_receipt_acceptance._client", side_effect=(first_client, resumed_client)), patch("scripts.phase103_consumer_receipt_acceptance.market_data_view_from_stream", return_value=SimpleNamespace()), patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view"), + patch( + "scripts.phase103_consumer_receipt_acceptance._strict_snapshot_for_c2", + return_value=current, + ) as readback, + ): + result = await _stream_resume( + product, + identity=SimpleNamespace(), + primary_url="https://query-primary", + secondary_url="https://query-secondary", + grpc_target="stream:8210", + state_dir=Path(raw), + timeout_seconds=1.0, + ) + + self.assertEqual(result, (10_005, 10_006, (), ())) + self.assertEqual(first_session.acknowledged, [first]) + self.assertEqual(resumed_session.acknowledged, [resumed[0]]) + readback.assert_awaited_once() + + async def test_non_execution_bar_can_be_quiet_only_with_current_signed_controls_and_final_reads(self): + product = self._product() + requirement = self._requirement() + strict_view = SimpleNamespace( + payload=SimpleNamespace(open_time_ns=20 * self.INTERVAL_NS) + ) + seed_view = SimpleNamespace(payload=SimpleNamespace(open_time_ns=18 * self.INTERVAL_NS)) + strict_warmup = SimpleNamespace(data=[strict_view], watermark_offset=40) + seed_warmup = SimpleNamespace(data=[seed_view], watermark_offset=38) + controls = ( + ControlEvent("REPLAYING", "retained BAR replay accepted"), + ControlEvent("LIVE", "stream live"), + ) + first_session = self._Session(warmup=seed_warmup, items=controls, quiet=True) + resumed_session = self._Session(warmup=seed_warmup, items=controls, quiet=True) + first_current = SimpleNamespace(data=SimpleNamespace()) + resumed_current = SimpleNamespace(data=SimpleNamespace()) + first_client = self._Client( + strict_warmup=strict_warmup, + session=first_session, + snapshots=(first_current,), + ) + resumed_client = self._Client( + strict_warmup=strict_warmup, + session=resumed_session, + snapshots=(resumed_current,), + ) + + with tempfile.TemporaryDirectory(prefix="qdl-c2-quiet-historical-bar-") as raw: + with ( + patch("scripts.phase103_consumer_receipt_acceptance.sdk_requirement", return_value=requirement), + patch("scripts.phase103_consumer_receipt_acceptance._client", side_effect=(first_client, resumed_client)), + patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view") as validate, + ): + result = await _stream_resume( + product, + identity=SimpleNamespace(), + primary_url="https://query-primary", + secondary_url="https://query-secondary", + grpc_target="stream:8210", + state_dir=Path(raw), + timeout_seconds=0.01, + ) + + self.assertEqual( + result, + ( + None, + None, + ("REPLAYING", "LIVE", "REPLAYING", "LIVE"), + ("CURRENT_FINAL_BAR", "CURRENT_FINAL_BAR"), + ), + ) + self.assertEqual( + _stream_handoff_mode( + product, + acknowledged_offset=result[0], + resumed_offset=result[1], + no_event_sessions=result[3], + ), + "CURRENT_FINAL_BAR_OBSERVED_NO_CURSOR", + ) + self.assertEqual(first_client.snapshot_calls, [requirement]) + self.assertEqual(resumed_client.snapshot_calls, [requirement]) + self.assertEqual(first_client.warmup_calls, []) + self.assertEqual(resumed_client.warmup_calls, []) + self.assertEqual(first_session.acknowledged, []) + self.assertEqual(resumed_session.acknowledged, []) + self.assertEqual(first_client.stream_calls, [(requirement, False)]) + self.assertEqual(resumed_client.stream_calls, [(requirement, False)]) + self.assertEqual( + validate.call_args_list, + [ + call(product, first_current.data), + call(product, resumed_current.data), + ], + ) + + async def test_quiet_final_bar_then_reopened_event_is_a_valid_no_cursor_handoff(self): + product = self._product() + requirement = self._requirement() + strict_view = SimpleNamespace( + payload=SimpleNamespace(open_time_ns=20 * self.INTERVAL_NS) + ) + seed_view = SimpleNamespace( + payload=SimpleNamespace(open_time_ns=18 * self.INTERVAL_NS) + ) + strict_warmup = SimpleNamespace(data=[strict_view], watermark_offset=40) + seed_warmup = SimpleNamespace(data=[seed_view], watermark_offset=38) + controls = ( + ControlEvent("REPLAYING", "retained BAR replay accepted"), + ControlEvent("LIVE", "stream live"), + ) + first_session = self._Session(warmup=seed_warmup, items=controls, quiet=True) + reopened = StreamEvent(41, "final-bar-after-reopen", object()) + resumed_session = self._Session( + warmup=seed_warmup, + items=controls + (reopened,), + ) + first_current = SimpleNamespace(data=SimpleNamespace()) + first_client = self._Client( + strict_warmup=strict_warmup, + session=first_session, + snapshots=(first_current,), + ) + resumed_client = self._Client( + strict_warmup=strict_warmup, + session=resumed_session, + ) + + with tempfile.TemporaryDirectory(prefix="qdl-c2-quiet-bar-reopen-") as raw: + with ( + patch( + "scripts.phase103_consumer_receipt_acceptance.sdk_requirement", + return_value=requirement, + ), + patch( + "scripts.phase103_consumer_receipt_acceptance._client", + side_effect=(first_client, resumed_client), + ), + patch( + "scripts.phase103_consumer_receipt_acceptance.market_data_view_from_stream", + return_value=SimpleNamespace(), + ), + patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view") as validate, + ): + result = await _stream_resume( + product, + identity=SimpleNamespace(), + primary_url="https://query-primary", + secondary_url="https://query-secondary", + grpc_target="stream:8210", + state_dir=Path(raw), + timeout_seconds=0.01, + ) + + self.assertEqual( + result, + ( + None, + None, + ("REPLAYING", "LIVE", "REPLAYING", "LIVE"), + ("CURRENT_FINAL_BAR", "EVENT_AFTER_REOPEN"), + ), + ) + self.assertEqual( + _stream_handoff_mode( + product, + acknowledged_offset=result[0], + resumed_offset=result[1], + no_event_sessions=result[3], + ), + "LIVE_EVENT_AFTER_REOPEN_NO_CURSOR", + ) + self.assertEqual(resumed_session.acknowledged, [reopened]) + self.assertEqual(first_client.snapshot_calls, [requirement]) + self.assertEqual(resumed_client.snapshot_calls, []) + self.assertEqual( + validate.call_args_list, + [ + call(product, first_current.data), + call(product, ANY), + ], + ) + + async def test_non_execution_quiet_bar_requires_signed_controls_before_current_read(self): + product = self._product() + requirement = self._requirement() + strict_view = SimpleNamespace( + payload=SimpleNamespace(open_time_ns=20 * self.INTERVAL_NS) + ) + strict_warmup = SimpleNamespace(data=[strict_view], watermark_offset=40) + seed_warmup = SimpleNamespace( + data=[SimpleNamespace(payload=SimpleNamespace(open_time_ns=18 * self.INTERVAL_NS))], + watermark_offset=38, + ) + client = self._Client( + strict_warmup=strict_warmup, + session=self._Session(warmup=seed_warmup, items=(), quiet=True), + snapshots=(SimpleNamespace(data=SimpleNamespace()),), + ) + + with tempfile.TemporaryDirectory(prefix="qdl-c2-quiet-historical-bar-controls-") as raw: + with ( + patch("scripts.phase103_consumer_receipt_acceptance.sdk_requirement", return_value=requirement), + patch("scripts.phase103_consumer_receipt_acceptance._client", return_value=client), + patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view"), + self.assertRaisesRegex(ContinuityError, "signed cursor stream"), + ): + await _stream_resume( + product, + identity=SimpleNamespace(), + primary_url="https://query-primary", + secondary_url="https://query-secondary", + grpc_target="stream:8210", + state_dir=Path(raw), + timeout_seconds=0.01, + ) + + self.assertEqual(client.snapshot_calls, []) + + async def test_non_execution_quiet_bar_allows_paced_open_before_post_open_window(self): + product = self._product() + requirement = self._requirement() + strict_view = SimpleNamespace( + payload=SimpleNamespace(open_time_ns=20 * self.INTERVAL_NS) + ) + strict_warmup = SimpleNamespace(data=[strict_view], watermark_offset=40) + seed_warmup = SimpleNamespace( + data=[SimpleNamespace(payload=SimpleNamespace(open_time_ns=18 * self.INTERVAL_NS))], + watermark_offset=38, + ) + controls = ( + ControlEvent("REPLAYING", "retained BAR replay accepted"), + ControlEvent("LIVE", "stream live"), + ) + first_session = self._Session( + warmup=seed_warmup, + items=controls, + quiet=True, + first_item_delay_seconds=0.02, + ) + resumed_session = self._Session( + warmup=seed_warmup, + items=controls, + quiet=True, + first_item_delay_seconds=0.02, + ) + first_client = self._Client( + strict_warmup=strict_warmup, + session=first_session, + snapshots=(SimpleNamespace(data=SimpleNamespace()),), + ) + resumed_client = self._Client( + strict_warmup=strict_warmup, + session=resumed_session, + snapshots=(SimpleNamespace(data=SimpleNamespace()),), + ) + + with tempfile.TemporaryDirectory(prefix="qdl-c2-paced-quiet-bar-") as raw: + with ( + patch("scripts.phase103_consumer_receipt_acceptance.sdk_requirement", return_value=requirement), + patch("scripts.phase103_consumer_receipt_acceptance._client", side_effect=(first_client, resumed_client)), + patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view"), + ): + result = await _stream_resume( + product, + identity=SimpleNamespace(), + primary_url="https://query-primary", + secondary_url="https://query-secondary", + grpc_target="stream:8210", + state_dir=Path(raw), + timeout_seconds=0.01, + stream_open_timeout_seconds=0.05, + ) + + self.assertEqual( + result, + ( + None, + None, + ("REPLAYING", "LIVE", "REPLAYING", "LIVE"), + ("CURRENT_FINAL_BAR", "CURRENT_FINAL_BAR"), + ), + ) + + async def test_non_execution_quiet_bar_keeps_post_open_control_bound(self): + product = self._product() + requirement = self._requirement() + strict_view = SimpleNamespace( + payload=SimpleNamespace(open_time_ns=20 * self.INTERVAL_NS) + ) + strict_warmup = SimpleNamespace(data=[strict_view], watermark_offset=40) + seed_warmup = SimpleNamespace( + data=[SimpleNamespace(payload=SimpleNamespace(open_time_ns=18 * self.INTERVAL_NS))], + watermark_offset=38, + ) + client = self._Client( + strict_warmup=strict_warmup, + session=self._Session( + warmup=seed_warmup, + items=(ControlEvent("REPLAYING", "accepted"),), + quiet=True, + first_item_delay_seconds=0.02, + ), + ) + + with tempfile.TemporaryDirectory(prefix="qdl-c2-paced-incomplete-controls-") as raw: + with ( + patch("scripts.phase103_consumer_receipt_acceptance.sdk_requirement", return_value=requirement), + patch("scripts.phase103_consumer_receipt_acceptance._client", return_value=client), + patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view"), + self.assertRaisesRegex(ContinuityError, "signed cursor stream"), + ): + await _stream_resume( + product, + identity=SimpleNamespace(), + primary_url="https://query-primary", + secondary_url="https://query-secondary", + grpc_target="stream:8210", + state_dir=Path(raw), + timeout_seconds=0.01, + stream_open_timeout_seconds=0.05, + ) + + self.assertEqual(client.snapshot_calls, []) + + async def test_non_execution_quiet_bar_rejects_failed_current_read(self): + product = self._product() + requirement = self._requirement() + strict_view = SimpleNamespace( + payload=SimpleNamespace(open_time_ns=20 * self.INTERVAL_NS) + ) + strict_warmup = SimpleNamespace(data=[strict_view], watermark_offset=40) + seed_warmup = SimpleNamespace( + data=[SimpleNamespace(payload=SimpleNamespace(open_time_ns=18 * self.INTERVAL_NS))], + watermark_offset=38, + ) + controls = ( + ControlEvent("REPLAYING", "retained BAR replay accepted"), + ControlEvent("LIVE", "stream live"), + ) + client = self._Client( + strict_warmup=strict_warmup, + session=self._Session(warmup=seed_warmup, items=controls, quiet=True), + ) + + with tempfile.TemporaryDirectory(prefix="qdl-c2-quiet-historical-bar-current-") as raw: + with ( + patch("scripts.phase103_consumer_receipt_acceptance.sdk_requirement", return_value=requirement), + patch("scripts.phase103_consumer_receipt_acceptance._client", return_value=client), + patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view"), + patch( + "scripts.phase103_consumer_receipt_acceptance._strict_snapshot_for_c2", + side_effect=ContinuityError("DATA_STALE", "current historical BAR stale"), + ), + self.assertRaisesRegex(ContinuityError, "current historical BAR stale"), + ): + await _stream_resume( + product, + identity=SimpleNamespace(), + primary_url="https://query-primary", + secondary_url="https://query-secondary", + grpc_target="stream:8210", + state_dir=Path(raw), + timeout_seconds=0.01, + ) + + async def test_final_bar_event_then_quiet_resume_does_not_wait_for_next_interval(self): + product = self._product() + requirement = self._requirement() + warmup = SimpleNamespace(data=[SimpleNamespace()], watermark_offset=10) + first = StreamEvent(11, "signed-11", object()) + controls = (ControlEvent("REPLAYING", "accepted"), ControlEvent("LIVE", "live")) + first_session = self._Session(warmup=warmup, items=(first,)) + resumed_session = self._Session(warmup=warmup, items=controls, quiet=True) + first_client = self._Client(strict_warmup=warmup, session=first_session) + resumed_client = self._Client(strict_warmup=warmup, session=resumed_session) + with ( + tempfile.TemporaryDirectory() as raw, + patch("scripts.phase103_consumer_receipt_acceptance.sdk_requirement", return_value=requirement), + patch("scripts.phase103_consumer_receipt_acceptance._client", side_effect=(first_client, resumed_client)), + patch("scripts.phase103_consumer_receipt_acceptance.market_data_view_from_stream", return_value=SimpleNamespace()), + patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view"), + patch("scripts.phase103_consumer_receipt_acceptance._strict_snapshot_for_c2", + return_value=SimpleNamespace(data=SimpleNamespace())) as current, + ): + result = await _stream_resume( + product, identity=SimpleNamespace(), primary_url="https://primary", + secondary_url="https://secondary", grpc_target="stream:8210", + state_dir=Path(raw), timeout_seconds=0.01, stream_open_timeout_seconds=0.1, + ) + self.assertEqual(result[:2], (None, None)) + self.assertEqual(result[3], ("CURSOR_ACKNOWLEDGED", "CURRENT_FINAL_BAR_AFTER_CURSOR")) + self.assertEqual(first_session.acknowledged, [first]) + self.assertEqual(resumed_client.stream_calls, [(requirement, True)]) + current.assert_awaited_once() + self.assertEqual(_stream_handoff_mode(product, acknowledged_offset=None, + resumed_offset=None, no_event_sessions=result[3]), + "SIGNED_CURSOR_REOPENED_NO_NEW_EVENT") + + async def test_execution_bar_keeps_the_live_stream_requirement(self): + product = self._product(grade=Grade.EXECUTION) + requirement = self._requirement(grade=Grade.EXECUTION) + warmup = SimpleNamespace( + data=[SimpleNamespace(payload=SimpleNamespace(open_time_ns=self.INTERVAL_NS))], + watermark_offset=10, + ) + first = StreamEvent(10, "resume-10", object()) + resumed = StreamEvent(11, "resume-11", object()) + first_session = self._Session(warmup=warmup, items=(first,)) + resumed_session = self._Session(warmup=warmup, items=(resumed,)) + first_client = self._Client(strict_warmup=warmup, session=first_session) + resumed_client = self._Client(strict_warmup=warmup, session=resumed_session) + + with tempfile.TemporaryDirectory(prefix="qdl-c2-live-bar-") as raw: + with ( + patch("scripts.phase103_consumer_receipt_acceptance.sdk_requirement", return_value=requirement), + patch("scripts.phase103_consumer_receipt_acceptance._client", side_effect=(first_client, resumed_client)), + patch("scripts.phase103_consumer_receipt_acceptance.market_data_view_from_stream", return_value=SimpleNamespace()), + patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view"), + patch( + "scripts.phase103_consumer_receipt_acceptance._strict_snapshot_for_c2", + return_value=SimpleNamespace(data=SimpleNamespace()), + ) as readback, + ): + result = await _stream_resume( + product, + identity=SimpleNamespace(), + primary_url="https://query-primary", + secondary_url="https://query-secondary", + grpc_target="stream:8210", + state_dir=Path(raw), + timeout_seconds=1.0, + ) + + self.assertEqual(result, (10, 11, (), ())) + self.assertEqual(first_client.warmup_calls, []) + self.assertEqual(first_client.stream_calls, [(requirement, False)]) + self.assertEqual(resumed_client.stream_calls, [(requirement, True)]) + readback.assert_awaited_once() + + async def test_execution_quote_stale_first_frame_requires_current_snapshot_before_resume(self): + requirement = DataRequirement( + instrument_uid="a953e16e-7138-5562-b5e8-c337a44d0b65", + feed=Feed.QUOTE, + consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", + warmup_limit=0, + max_freshness_ms=3_000, + stale_policy=StalePolicy.BLOCK, + ) + product = SimpleNamespace( + delivery=DeliveryClass.DURABLE, + feed=Feed.QUOTE, + interval=None, + requirement=SimpleNamespace( + consumer_grade=Grade.EXECUTION, + max_freshness_ms=3_000, + ), + identity=("trading-system.paper.stable", "instrument", "QUOTE", "", "policy"), + ) + warmup = SimpleNamespace(data=[object()], watermark_offset=9) + first = StreamEvent(10, "resume-10", object()) + resumed = StreamEvent(11, "resume-11", object()) + first_session = self._Session(warmup=warmup, items=(first,)) + resumed_session = self._Session(warmup=warmup, items=(resumed,)) + first_client = self._Client(strict_warmup=warmup, session=first_session) + resumed_client = self._Client(strict_warmup=warmup, session=resumed_session) + first_current = SimpleNamespace(data=SimpleNamespace()) + resumed_current = SimpleNamespace(data=SimpleNamespace()) + + with tempfile.TemporaryDirectory(prefix="qdl-c2-stale-quote-") as raw: + with ( + patch("scripts.phase103_consumer_receipt_acceptance.sdk_requirement", return_value=requirement), + patch("scripts.phase103_consumer_receipt_acceptance._client", side_effect=(first_client, resumed_client)), + patch( + "scripts.phase103_consumer_receipt_acceptance.market_data_view_from_stream", + side_effect=( + ContinuityError("DATA_STALE", "delayed quote"), + SimpleNamespace(), + SimpleNamespace(), + ), + ) as project, + patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view") as validate, + patch( + "scripts.phase103_consumer_receipt_acceptance._strict_snapshot_for_c2", + side_effect=(first_current, resumed_current), + ) as readback, + ): + result = await _stream_resume( + product, + identity=SimpleNamespace(), + primary_url="https://query-primary", + secondary_url="https://query-secondary", + grpc_target="stream:8210", + state_dir=Path(raw), + timeout_seconds=1.0, + ) + + self.assertEqual(result, (10, 11, (), ())) + self.assertEqual(first_session.acknowledged, [first]) + self.assertEqual(resumed_session.acknowledged, [resumed]) + self.assertEqual(readback.await_count, 2) + self.assertNotIn("replay_only", project.call_args_list[0].kwargs) + self.assertTrue(project.call_args_list[1].kwargs["replay_only"]) + self.assertTrue(project.call_args_list[2].kwargs["replay_only"]) + self.assertIn( + call(product, first_current.data), + validate.call_args_list, + ) + self.assertIn( + call(product, resumed_current.data), + validate.call_args_list, + ) + + async def test_execution_quote_stale_first_frame_fails_closed_when_current_snapshot_is_stale(self): + requirement = DataRequirement( + instrument_uid="a953e16e-7138-5562-b5e8-c337a44d0b65", + feed=Feed.QUOTE, + consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", + warmup_limit=0, + max_freshness_ms=3_000, + stale_policy=StalePolicy.BLOCK, + ) + product = SimpleNamespace( + delivery=DeliveryClass.DURABLE, + feed=Feed.QUOTE, + interval=None, + requirement=SimpleNamespace( + consumer_grade=Grade.EXECUTION, + max_freshness_ms=3_000, + ), + identity=("trading-system.paper.stable", "instrument", "QUOTE", "", "policy"), + ) + warmup = SimpleNamespace(data=[object()], watermark_offset=9) + first = StreamEvent(10, "resume-10", object()) + first_client = self._Client( + strict_warmup=warmup, + session=self._Session(warmup=warmup, items=(first,)), + ) + + with tempfile.TemporaryDirectory(prefix="qdl-c2-stale-quote-block-") as raw: + with ( + patch("scripts.phase103_consumer_receipt_acceptance.sdk_requirement", return_value=requirement), + patch("scripts.phase103_consumer_receipt_acceptance._client", return_value=first_client), + patch( + "scripts.phase103_consumer_receipt_acceptance.market_data_view_from_stream", + side_effect=( + ContinuityError("DATA_STALE", "delayed quote"), + SimpleNamespace(), + ), + ), + patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view"), + patch( + "scripts.phase103_consumer_receipt_acceptance._strict_snapshot_for_c2", + side_effect=ContinuityError("DATA_STALE", "current quote stale"), + ), + self.assertRaisesRegex(ContinuityError, "current quote stale"), ): - result = await _stream_resume( + await _stream_resume( product, identity=SimpleNamespace(), primary_url="https://query-primary", @@ -419,27 +1139,87 @@ async def test_non_execution_bar_drains_bounded_replay_to_strict_watermark(self) timeout_seconds=1.0, ) - self.assertEqual(result, (39, 42, (), ())) - self.assertEqual(first_session.acknowledged, [first]) - self.assertEqual(resumed_session.acknowledged, list(resumed)) - async def test_execution_bar_keeps_the_live_stream_requirement(self): - product = self._product(grade=Grade.EXECUTION) - requirement = self._requirement(grade=Grade.EXECUTION) - warmup = SimpleNamespace(data=[SimpleNamespace(payload=SimpleNamespace(open_time_ns=self.INTERVAL_NS))]) - first = StreamEvent(10, "resume-10", object()) - resumed = StreamEvent(11, "resume-11", object()) - first_session = self._Session(warmup=warmup, items=(first,)) - resumed_session = self._Session(warmup=warmup, items=(resumed,)) - first_client = self._Client(strict_warmup=warmup, session=first_session) - resumed_client = self._Client(strict_warmup=warmup, session=resumed_session) +class Phase103ReplayReadbackTests(unittest.IsolatedAsyncioTestCase): + class _Session: + def __init__(self, *, watermark_offset: int, items): + self.warmup = SimpleNamespace(data=[object()], watermark_offset=watermark_offset) + self._items = iter(items) + self.acknowledged = [] - with tempfile.TemporaryDirectory(prefix="qdl-c2-live-bar-") as raw: + async def __anext__(self): + try: + return next(self._items) + except StopIteration as error: + raise StopAsyncIteration from error + + def acknowledge(self, event): + self.acknowledged.append(event) + + class _Context: + def __init__(self, session): + self.session = session + + async def __aenter__(self): + return self.session + + async def __aexit__(self, exc_type, exc, traceback): + return False + + class _Client: + def __init__(self, session): + self.session = session + self.stream_calls = [] + self.closed = False + + def warmup_then_stream(self, requirement, *, resume_restored_state=False): + self.stream_calls.append((requirement, resume_restored_state)) + return Phase103ReplayReadbackTests._Context(self.session) + + async def close(self): + self.closed = True + + async def test_resumed_cursor_frame_requires_a_fresh_readback_before_receipt(self): + requirement = DataRequirement( + instrument_uid="a953e16e-7138-5562-b5e8-c337a44d0b65", + feed=Feed.QUOTE, + consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", + max_freshness_ms=2_000, + stale_policy=StalePolicy.BLOCK, + ) + product = SimpleNamespace( + delivery=DeliveryClass.DURABLE, + feed=Feed.QUOTE, + interval=None, + requirement=SimpleNamespace( + consumer_grade=Grade.EXECUTION, + max_freshness_ms=2_000, + ), + identity=("trading-system.paper.stable", "instrument", "QUOTE", "", "policy"), + ) + first = StreamEvent(10, "token-10", object()) + replayed = StreamEvent(11, "token-11", object()) + first_session = self._Session(watermark_offset=9, items=(first,)) + # Even an offset after the fresh snapshot watermark is state recovery + # in the bounded restored-cursor probe, never a live execution price. + resumed_session = self._Session(watermark_offset=10, items=(replayed,)) + first_client = self._Client(first_session) + resumed_client = self._Client(resumed_session) + current = SimpleNamespace(data=SimpleNamespace()) + projected = [] + + def project(event, *, template, requirement, **kwargs): + projected.append((event.logical_offset, kwargs)) + return SimpleNamespace() + + with tempfile.TemporaryDirectory(prefix="qdl-c2-replay-readback-") as raw: with ( patch("scripts.phase103_consumer_receipt_acceptance.sdk_requirement", return_value=requirement), patch("scripts.phase103_consumer_receipt_acceptance._client", side_effect=(first_client, resumed_client)), - patch("scripts.phase103_consumer_receipt_acceptance.market_data_view_from_stream", return_value=SimpleNamespace()), - patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view"), + patch("scripts.phase103_consumer_receipt_acceptance.market_data_view_from_stream", side_effect=project), + patch("scripts.phase103_consumer_receipt_acceptance._strict_snapshot_for_c2", return_value=current) as readback, + patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view") as validate, ): result = await _stream_resume( product, @@ -452,15 +1232,24 @@ async def test_execution_bar_keeps_the_live_stream_requirement(self): ) self.assertEqual(result, (10, 11, (), ())) - self.assertEqual(first_client.warmup_calls, []) - self.assertEqual(first_client.stream_calls, [(requirement, False)]) - self.assertEqual(resumed_client.stream_calls, [(requirement, True)]) + self.assertEqual(projected, [(10, {}), (11, {"replay_only": True})]) + readback.assert_awaited_once() + self.assertEqual(first_session.acknowledged, [first]) + self.assertEqual(resumed_session.acknowledged, [replayed]) + self.assertEqual( + validate.call_args_list, + [ + call(product, ANY, require_current_quality=True), + call(product, ANY, require_current_quality=False, state_replay=True), + call(product, current.data), + ], + ) class Phase103QuietTradeStreamTests(unittest.IsolatedAsyncioTestCase): class _Session: def __init__(self, *, items=(), quiet: bool = False): - self.warmup = SimpleNamespace(data=[object()]) + self.warmup = SimpleNamespace(data=[object()], watermark_offset=0) self._items = iter(items) self._quiet = quiet self.acknowledged = [] @@ -827,7 +1616,7 @@ async def test_checkpointed_trade_reopen_without_new_print_is_explicit(self): "SIGNED_CURSOR_REOPENED_NO_NEW_EVENT", ) - async def test_delivered_observed_trade_keeps_strict_cursor_replay(self): + async def test_delivered_observed_trade_uses_state_replay_then_strict_readback(self): requirement = self._requirement() product = self._product(requirement) first = StreamEvent(10, "resume-10", object()) @@ -852,6 +1641,10 @@ async def test_delivered_observed_trade_keeps_strict_cursor_replay(self): return_value=SimpleNamespace(), ), patch("scripts.phase103_consumer_receipt_acceptance.validate_product_view"), + patch( + "scripts.phase103_consumer_receipt_acceptance._strict_snapshot_for_c2", + return_value=SimpleNamespace(data=SimpleNamespace()), + ) as readback, ): result = await _stream_resume( product, @@ -876,6 +1669,7 @@ async def test_delivered_observed_trade_keeps_strict_cursor_replay(self): self.assertEqual(first_session.acknowledged, [first]) self.assertEqual(second_session.acknowledged, [resumed]) self.assertEqual(second_client.stream_calls, [(requirement, True)]) + readback.assert_awaited_once() async def test_quiet_trade_rejects_disconnected_stale_session_gap_and_wrong_policy(self): requirement = self._requirement() @@ -992,7 +1786,8 @@ def _status( class _Client: def __init__(self, snapshots, status): self._snapshots = iter(snapshots) - self._status = status + self._statuses = iter(status) if isinstance(status, tuple) else None + self._status = status[-1] if isinstance(status, tuple) else status self.snapshot_calls = 0 self.status_calls = 0 @@ -1005,6 +1800,11 @@ async def snapshot(self, _requirement): async def feed_status(self, _requirement): self.status_calls += 1 + if self._statuses is not None: + try: + return next(self._statuses) + except StopIteration: + pass return self._status class _StreamContext: @@ -1073,24 +1873,208 @@ async def test_fresh_quote_status_after_a_stale_snapshot_retries_under_the_same_ self.assertEqual(client.snapshot_calls, 2) self.assertEqual(client.status_calls, 1) - async def test_disconnected_quote_never_retries_or_accepts_stale_data(self): + async def test_fresh_trade_without_session_sla_retries_only_after_typed_current_status(self): + requirement = DataRequirement( + instrument_uid="a953e16e-7138-5562-b5e8-c337a44d0b65", + feed=Feed.TRADE, + consumer_grade=Grade.RESEARCH, + source_policy_id="crypto_primary_v2", + max_freshness_ms=15_000, + stale_policy=StalePolicy.BLOCK, + ) + product = SimpleNamespace(feed=Feed.TRADE) + client = self._Client( + (DataLayerError("DATA_STALE", "late projected trade"), "fresh-trade"), + self._status( + requirement, + state="LIVE", + event_recency_state="LIVE", + freshness_ms=1, + provider_session_state="NOT_APPLICABLE", + provider_session_liveness_ms=None, + execution_eligible=True, + ), + ) + + result = await _strict_snapshot_for_c2( + client, + product=product, + requirement=requirement, + timeout_seconds=0.25, + ) + + self.assertEqual(result, "fresh-trade") + self.assertEqual(client.snapshot_calls, 2) + self.assertEqual(client.status_calls, 1) + + async def test_trade_without_session_sla_rejects_quiet_or_disconnected_status(self): + requirement = DataRequirement( + instrument_uid="a953e16e-7138-5562-b5e8-c337a44d0b65", + feed=Feed.TRADE, + consumer_grade=Grade.RESEARCH, + source_policy_id="crypto_primary_v2", + max_freshness_ms=15_000, + stale_policy=StalePolicy.BLOCK, + ) + product = SimpleNamespace(feed=Feed.TRADE) + for status in ( + self._status( + requirement, + state="STALE", + event_recency_state="STALE", + provider_session_state="NOT_APPLICABLE", + provider_session_liveness_ms=None, + execution_eligible=False, + ), + self._status( + requirement, + state="LIVE", + event_recency_state="LIVE", + freshness_ms=1, + provider_session_state="DISCONNECTED", + provider_session_liveness_ms=None, + execution_eligible=True, + ), + ): + client = self._Client((DataLayerError("DATA_STALE", "late trade"),), status) + with self.subTest(status=status.quality.provider_session_state), self.assertRaisesRegex( + ContinuityError, "fresh executable status" + ): + await _strict_snapshot_for_c2( + client, + product=product, + requirement=requirement, + timeout_seconds=0.25, + ) + self.assertEqual(client.snapshot_calls, 1) + self.assertEqual(client.status_calls, 1) + + async def test_strict_snapshot_preserves_the_typed_sdk_envelope(self): requirement = self._requirement() + expected = SimpleNamespace(kind="market-data-view") client = self._Client( - (DataLayerError("DATA_STALE", "stale BBO"),), + (SnapshotResponse.model_construct(request_id="test", data=expected),), + self._status( + requirement, + state="LIVE", + event_recency_state="LIVE", + freshness_ms=1, + execution_eligible=True, + ), + ) + + result = await _strict_snapshot_for_c2( + client, + product=self._product(), + requirement=requirement, + timeout_seconds=0.25, + ) + + self.assertIs(result.data, expected) + self.assertEqual(client.snapshot_calls, 1) + + async def test_persistently_disconnected_quote_times_out_without_accepting_stale_data(self): + requirement = self._requirement() + client = self._Client( + tuple(DataLayerError("DATA_STALE", "stale BBO") for _ in range(32)), self._status(requirement, provider_session_state="DISCONNECTED"), ) - with self.assertRaisesRegex(ContinuityError, "live provider session"): + with patch( + "scripts.phase103_consumer_receipt_acceptance._TRANSIENT_SESSION_RETRY_SECONDS", + 0.001, + ), self.assertRaisesRegex(ContinuityError, "before its deadline"): await _strict_snapshot_for_c2( + client, + product=self._product(), + requirement=requirement, + timeout_seconds=0.01, + ) + + self.assertGreater(client.snapshot_calls, 1) + self.assertGreater(client.status_calls, 1) + + async def test_transitional_quote_session_retries_then_requires_a_fresh_snapshot(self): + requirement = self._requirement() + client = self._Client( + (DataLayerError("DATA_STALE", "session reconnecting"), "fresh-snapshot"), + self._status(requirement, provider_session_state="DISCONNECTED"), + ) + + with patch( + "scripts.phase103_consumer_receipt_acceptance._TRANSIENT_SESSION_RETRY_SECONDS", + 0.001, + ): + result = await _strict_snapshot_for_c2( client, product=self._product(), requirement=requirement, timeout_seconds=0.25, ) - self.assertEqual(client.snapshot_calls, 1) + self.assertEqual(result, "fresh-snapshot") + self.assertEqual(client.snapshot_calls, 2) + self.assertEqual(client.status_calls, 1) + + async def test_transitional_book_delta_session_retries_without_accepting_a_stale_delta(self): + requirement = DataRequirement( + instrument_uid="6c7c9256-2905-5c75-a149-fa0ac36bbbc7", + feed=Feed.BOOK_DELTA, + consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", + max_freshness_ms=2_000, + max_session_liveness_ms=45_000, + stale_policy=StalePolicy.BLOCK, + ) + product = SimpleNamespace(feed=Feed.BOOK_DELTA, delivery=DeliveryClass.DURABLE) + client = self._Client( + (DataLayerError("DATA_STALE", "book session reconnecting"), "fresh-book-delta"), + self._status(requirement, provider_session_state="UNKNOWN"), + ) + + with patch( + "scripts.phase103_consumer_receipt_acceptance._TRANSIENT_SESSION_RETRY_SECONDS", + 0.001, + ): + result = await _strict_snapshot_for_c2( + client, + product=product, + requirement=requirement, + timeout_seconds=0.25, + ) + + self.assertEqual(result, "fresh-book-delta") + self.assertEqual(client.snapshot_calls, 2) self.assertEqual(client.status_calls, 1) + async def test_transitional_session_rejects_gap_and_cross_symbol_status(self): + requirement = self._requirement() + for name, status in { + "gap": self._status( + requirement, + provider_session_state="DISCONNECTED", + gap_open=True, + ), + "cross_symbol": self._status( + requirement, + provider_session_state="DISCONNECTED", + ).model_copy(update={"instrument_uid": "other-instrument"}), + }.items(): + with self.subTest(status=name): + client = self._Client( + (DataLayerError("DATA_STALE", "stale BBO"),), + status, + ) + with self.assertRaisesRegex(ContinuityError, "live provider session"): + await _strict_snapshot_for_c2( + client, + product=self._product(), + requirement=requirement, + timeout_seconds=0.25, + ) + self.assertEqual(client.snapshot_calls, 1) + self.assertEqual(client.status_calls, 1) + async def test_quiet_quote_deadline_fails_closed_without_a_fresh_snapshot(self): requirement = self._requirement() client = self._Client( @@ -1109,6 +2093,183 @@ async def test_quiet_quote_deadline_fails_closed_without_a_fresh_snapshot(self): self.assertEqual(client.snapshot_calls, 1) self.assertEqual(client.status_calls, 1) + async def test_verified_book_snapshot_retries_only_until_a_fresh_renewal_arrives(self): + requirement = DataRequirement( + instrument_uid="6c7c9256-2905-5c75-a149-fa0ac36bbbc7", + feed=Feed.BOOK_SNAPSHOT, + consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", + max_freshness_ms=60_000, + stale_policy=StalePolicy.BLOCK, + ) + product = SimpleNamespace(feed=Feed.BOOK_SNAPSHOT) + client = self._Client( + (DataLayerError("DATA_STALE", "book snapshot renewal pending"), "fresh-book"), + self._status( + requirement, + provider_session_state="NOT_APPLICABLE", + provider_session_liveness_ms=None, + ), + ) + + with patch( + "scripts.phase103_consumer_receipt_acceptance._BOOK_SNAPSHOT_RETRY_SECONDS", + 0.001, + ): + result = await _strict_snapshot_for_c2( + client, + product=product, + requirement=requirement, + timeout_seconds=0.25, + ) + + self.assertEqual(result, "fresh-book") + self.assertEqual(client.snapshot_calls, 2) + self.assertEqual(client.status_calls, 1) + + async def test_book_snapshot_gap_or_session_mismatch_never_retries(self): + requirement = DataRequirement( + instrument_uid="6c7c9256-2905-5c75-a149-fa0ac36bbbc7", + feed=Feed.BOOK_SNAPSHOT, + consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", + max_freshness_ms=60_000, + stale_policy=StalePolicy.BLOCK, + ) + product = SimpleNamespace(feed=Feed.BOOK_SNAPSHOT) + for status in ( + self._status( + requirement, + provider_session_state="NOT_APPLICABLE", + provider_session_liveness_ms=None, + gap_open=True, + ), + self._status(requirement, provider_session_state="DISCONNECTED"), + ): + client = self._Client((DataLayerError("DATA_STALE", "bad book"),), status) + with self.assertRaisesRegex(ContinuityError, "complete, gap-free snapshot state"): + await _strict_snapshot_for_c2( + client, + product=product, + requirement=requirement, + timeout_seconds=0.25, + ) + self.assertEqual(client.snapshot_calls, 1) + self.assertEqual(client.status_calls, 1) + + async def test_session_backed_book_renewal_matrix(self): + product = SimpleNamespace(feed=Feed.BOOK_SNAPSHOT) + for venue in ("BINANCE", "OKX"): + for symbol in ("BTC", "ETH", "SOL", "DOGE", "BNB"): + for session_limit in (None, 3_000): + requirement = DataRequirement( + instrument_uid=f"{venue}-{symbol}", feed=Feed.BOOK_SNAPSHOT, + consumer_grade=Grade.EXECUTION, source_policy_id="crypto_primary_v2", + max_freshness_ms=3_000, max_session_liveness_ms=session_limit, + stale_policy=StalePolicy.BLOCK, + ) + for event_state in ("LIVE", "STALE"): + with self.subTest(venue=venue, symbol=symbol, limit=session_limit, + event_state=event_state): + client = self._Client( + (DataLayerError("DATA_STALE", "pending"), "fresh-book"), + self._status(requirement, state=event_state, + event_recency_state=event_state, + provider_session_liveness_ms=806), + ) + with patch("scripts.phase103_consumer_receipt_acceptance." + "_BOOK_SNAPSHOT_RETRY_SECONDS", 0.001): + result = await _strict_snapshot_for_c2( + client, product=product, requirement=requirement, + timeout_seconds=0.25, + ) + self.assertEqual(result, "fresh-book") + self.assertEqual(client.snapshot_calls, 2) + self.assertEqual(client.status_calls, 1) + + async def test_book_renewal_rejects_invalid_sessions_identity_and_quality(self): + requirement = replace(self._requirement(), feed=Feed.BOOK_SNAPSHOT) + base = self._status(requirement) + statuses = { + "disconnected": self._status(requirement, provider_session_state="DISCONNECTED"), + "unknown": self._status(requirement, provider_session_state="UNKNOWN"), + "stale": self._status(requirement, provider_session_state="STALE"), + "missing_age": self._status(requirement, provider_session_liveness_ms=None), + "expired_age": self._status(requirement, provider_session_liveness_ms=45_001), + "required_session_absent": self._status(requirement, + provider_session_state="NOT_APPLICABLE", provider_session_liveness_ms=None), + "gap": self._status(requirement, gap_open=True), + "partial": self._status(requirement, complete=False), + "offline": self._status(requirement, state="OFFLINE"), + "other_symbol": base.model_copy(update={"instrument_uid": "other"}), + "other_feed": base.model_copy(update={"feed": Feed.BOOK_DELTA}), + "other_policy": base.model_copy(update={ + "quality": base.quality.model_copy(update={"policy_id": "other"})}), + } + for name, status in statuses.items(): + with self.subTest(case=name): + client = self._Client((DataLayerError("DATA_STALE", "bad book"),), status) + with self.assertRaises(C2StatusEvidenceError): + await _strict_snapshot_for_c2( + client, product=SimpleNamespace(feed=Feed.BOOK_SNAPSHOT), + requirement=requirement, timeout_seconds=0.25, + ) + self.assertEqual(client.snapshot_calls, 1) + + async def test_book_renewal_never_accepts_status_instead_of_strict_snapshot(self): + requirement = replace(self._requirement(), feed=Feed.BOOK_SNAPSHOT) + client = self._Client((DataLayerError("DATA_STALE", "pending"),), + self._status(requirement)) + with self.assertRaisesRegex(ContinuityError, "before its deadline"): + await _strict_snapshot_for_c2( + client, product=SimpleNamespace(feed=Feed.BOOK_SNAPSHOT), + requirement=requirement, timeout_seconds=0.001, + ) + self.assertEqual(client.snapshot_calls, 1) + + async def test_book_snapshot_failure_retains_compact_typed_status_only(self): + requirement = DataRequirement( + instrument_uid="6c7c9256-2905-5c75-a149-fa0ac36bbbc7", + feed=Feed.BOOK_SNAPSHOT, + consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", + max_freshness_ms=60_000, + stale_policy=StalePolicy.BLOCK, + ) + product = SimpleNamespace(feed=Feed.BOOK_SNAPSHOT) + status = self._status( + requirement, + state="STALE", + event_recency_state="STALE", + provider_session_state="NOT_APPLICABLE", + provider_session_liveness_ms=None, + gap_open=True, + complete=False, + execution_eligible=False, + ) + client = self._Client((DataLayerError("DATA_STALE", "bad book"),), status) + + with self.assertRaises(C2StatusEvidenceError) as raised: + await _strict_snapshot_for_c2( + client, + product=product, + requirement=requirement, + timeout_seconds=0.25, + ) + + evidence = raised.exception.status_evidence + self.assertEqual(evidence["instrument_uid"], requirement.instrument_uid) + self.assertEqual(evidence["feed"], "BOOK_SNAPSHOT") + self.assertEqual(evidence["quality"]["state"], "STALE") + self.assertTrue(evidence["quality"]["gap_open"]) + self.assertFalse(evidence["quality"]["complete"]) + self.assertFalse(evidence["payload_recorded"]) + self.assertNotIn("levels", repr(evidence)) + + def test_compact_feed_status_rejects_untyped_status(self): + with self.assertRaisesRegex(ValueError, "instrument identity"): + compact_feed_status(SimpleNamespace()) + async def test_quiet_connected_trade_retries_but_still_requires_a_fresh_snapshot(self): requirement = DataRequirement( instrument_uid="6c7c9256-2905-5c75-a149-fa0ac36bbbc7", diff --git a/tests/test_phase104_reference_batch.py b/tests/test_phase104_reference_batch.py index 35cee722..15124462 100644 --- a/tests/test_phase104_reference_batch.py +++ b/tests/test_phase104_reference_batch.py @@ -136,6 +136,35 @@ async def fetch(self, request, *, capability, received_at_ns): ) +class ReceiptTimestampAdapter: + def __init__(self) -> None: + self.request_started_at_ns = None + + async def fetch(self, request, *, capability, received_at_ns): + self.request_started_at_ns = received_at_ns + observation = ReferenceObservation( + instrument_uid=request.instrument.instrument_uid, + instrument_revision=request.instrument.metadata_revision, + product=request.product, + observed_at_ns=150_000_000, + fields=(decimal_field("open_interest_contracts", "1", "CONTRACTS"),), + labels=(("native_symbol", request.instrument.native_symbol),), + ) + return ReferenceFetch( + observations=(observation,), + lineage=( + provider_lineage( + provider="BINANCE_DIRECT", + endpoint="test", + capability_name="open_interest", + capability=capability, + adapter_version="test", + ), + ), + coverage=ReferenceCoverage(0, 200, 150, 150, True, True, False, "TEST"), + ) + + class ReferenceRequestTests(unittest.TestCase): def setUp(self) -> None: self.btc = instrument( @@ -198,30 +227,48 @@ async def test_paginated_overlap_dedup_and_concurrent_symbol_isolation(self): self.assertTrue(btc_result.coverage.complete_right) self.assertEqual(sorted(calls), [("BTCUSDT", 100), ("BTCUSDT", 201), ("BTCUSDT", 301), ("ETHUSDT", 100), ("ETHUSDT", 201), ("ETHUSDT", 301)]) + async def test_successful_reference_result_receipt_is_captured_after_provider_fetch(self): + adapter = ReceiptTimestampAdapter() + values = iter((100_000_000, 200_000_000)) + batch = ReferenceBatch( + {("BINANCE", "USDM"): adapter}, + clock_ns=lambda: next(values), + ) + + result = await batch.fetch_one( + ReferenceRequest( + instrument=self.btc, + product=ReferenceProduct.OPEN_INTEREST, + ) + ) + + self.assertEqual(adapter.request_started_at_ns, 100_000_000) + self.assertEqual(result.received_at_ns, 200_000_000) + self.assertLessEqual(result.observations[0].observed_at_ns, result.received_at_ns) + async def test_funding_boundary_jitter_is_tolerated_without_rewriting_raw_time(self): calls = [] def funding(symbol, *, start_time, end_time, limit, **kwargs): - del end_time, limit, kwargs - calls.append((symbol, start_time)) - rows = { - 1_000: [ - {"symbol": symbol, "fundingRate": "0.1", "fundingTime": "1003"}, - {"symbol": symbol, "fundingRate": "0.2", "fundingTime": "2000"}, - ], - 2_001: [ - {"symbol": symbol, "fundingRate": "0.3", "fundingTime": "3003"}, - ], - } - return {"data": rows[start_time]} + del limit, kwargs + calls.append((symbol, start_time, end_time)) + rows = [ + {"symbol": symbol, "fundingRate": "0.1", "fundingTime": "100003"}, + {"symbol": symbol, "fundingRate": "0.2", "fundingTime": "200000"}, + ] + if end_time >= 300003: + rows.append( + {"symbol": symbol, "fundingRate": "0.3", "fundingTime": "300003"} + ) + return {"data": rows} result = await ReferenceBatch({("BINANCE", "USDM"): BinanceUsdmReferenceAdapter( funding_fetcher=funding, max_attempts=1, sleep=no_sleep, )}).fetch_one(ReferenceRequest( instrument=self.btc, product=ReferenceProduct.FUNDING_RATE, - start_ms=1_000, - end_ms=3_005, + start_ms=100_000, + end_ms=300_000, limit=3, page_size=2, max_pages=2, @@ -230,11 +277,12 @@ def funding(symbol, *, start_time, end_time, limit, **kwargs): self.assertEqual(result.status, ReferenceStatus.OK) self.assertEqual( [item.observed_at_ns // 1_000_000 for item in result.observations], - [1_003, 2_000, 3_003], + [100_003, 200_000, 300_003], ) self.assertTrue(result.coverage.complete_left) self.assertTrue(result.coverage.complete_right) - self.assertEqual(calls, [("BTCUSDT", 1_000), ("BTCUSDT", 2_001)]) + self.assertEqual(calls[0], ("BTCUSDT", 100_000, 360_000)) + self.assertTrue(all(end_time == 360_000 for _symbol, _start_time, end_time in calls)) async def test_funding_boundary_gap_beyond_tolerance_remains_partial(self): def funding(symbol, **kwargs): @@ -363,9 +411,21 @@ def basis(pair, contract_type, period, limit, start_time, end_time, **kwargs): self.assertTrue(taker_result.coverage.complete_left) self.assertTrue(taker_result.coverage.complete_right) self.assertFalse(taker_result.coverage.truncated) + self.assertEqual( + [item.observed_at_ns // 1_000_000 for item in taker_result.observations], + [2 * hour_ms - 1, 3 * hour_ms - 1], + ) + self.assertIn(("period_open_time_ms", str(hour_ms)), taker_result.observations[0].labels) + self.assertIn(("period_close_time_ms", str(2 * hour_ms - 1)), taker_result.observations[0].labels) self.assertTrue(basis_result.coverage.complete_left) self.assertTrue(basis_result.coverage.complete_right) self.assertFalse(basis_result.coverage.truncated) + self.assertEqual( + [item.observed_at_ns // 1_000_000 for item in basis_result.observations], + [2 * hour_ms - 1, 3 * hour_ms - 1], + ) + self.assertIn(("period_open_time_ms", str(hour_ms)), basis_result.observations[0].labels) + self.assertIn(("period_close_time_ms", str(2 * hour_ms - 1)), basis_result.observations[0].labels) async def test_taker_provider_window_advances_one_period_without_changing_logical_coverage(self): hour_ms = 3_600_000 @@ -398,7 +458,7 @@ def taker(symbol, period, limit, start_time, end_time, **kwargs): self.assertEqual(calls, [("BTCUSDT", 2, 2 * hour_ms, 4 * hour_ms - 1)]) self.assertEqual( [item.observed_at_ns // 1_000_000 for item in result.observations], - [hour_ms, 2 * hour_ms], + [2 * hour_ms - 1, 3 * hour_ms - 1], ) self.assertTrue(result.coverage.complete_left) self.assertTrue(result.coverage.complete_right) diff --git a/tests/test_phase105_consumer_acceptance.py b/tests/test_phase105_consumer_acceptance.py index f8a24f4b..3dc3250a 100644 --- a/tests/test_phase105_consumer_acceptance.py +++ b/tests/test_phase105_consumer_acceptance.py @@ -60,12 +60,11 @@ def test_scope_is_exactly_release_v2_primary_for_all_paper_classes(self): for item in scope.products } self.assertEqual(actual, expected) - self.assertEqual(len(scope.products), 149) + self.assertEqual(len(scope.products), 299) self.assertEqual( Counter(product.delivery for product in scope.products), { - DeliveryClass.DURABLE: 74, - DeliveryClass.PROVIDER_PASS_THROUGH: 10, + DeliveryClass.DURABLE: 234, DeliveryClass.ON_DEMAND: 65, }, ) @@ -102,13 +101,12 @@ def test_five_liquid_selection_is_exactly_the_full_v2_release_scope(self): (product.consumer_id, requirement_key(product.requirement)) for product in scope.products } - self.assertEqual(len(scope.products), 145) + self.assertEqual(len(scope.products), 295) self.assertEqual(actual, expected) self.assertEqual( Counter(product.delivery for product in scope.products), { - DeliveryClass.DURABLE: 70, - DeliveryClass.PROVIDER_PASS_THROUGH: 10, + DeliveryClass.DURABLE: 230, DeliveryClass.ON_DEMAND: 65, }, ) @@ -133,6 +131,45 @@ def test_paper_trade_routes_declare_observed_event_recency_and_session_sla(self) for item in trades )) + def test_paper_book_delta_routes_declare_observed_continuity_and_session_sla(self): + scope = build_release_consumer_acceptance_scope( + self.release, + catalog=self.catalog, + acquisition=self.acquisition, + consumer_ids=FIVE_LIQUID_CONSUMER_IDS, + ) + deltas = [item for item in scope.products if item.feed is FeedType.BOOK_DELTA] + self.assertEqual(len(deltas), 20) + self.assertTrue(all( + item.requirement.event_recency_policy is StalePolicy.OBSERVE + and item.requirement.max_session_liveness_ms == 45_000 + and item.requirement.max_freshness_ms == 2_000 + and item.requirement.stale_policy is StalePolicy.BLOCK + and item.requirement.gap_policy.value == "BLOCK" + for item in deltas + )) + self.assertEqual( + { + ( + self.catalog.instrument_for(item.instrument_uid).identity.venue, + self.catalog.instrument_for(item.instrument_uid).native_symbol, + ) + for item in deltas + }, + { + ("BINANCE", "BTCUSDT"), + ("BINANCE", "ETHUSDT"), + ("BINANCE", "SOLUSDT"), + ("BINANCE", "DOGEUSDT"), + ("BINANCE", "BNBUSDT"), + ("OKX", "BTC-USDT-SWAP"), + ("OKX", "ETH-USDT-SWAP"), + ("OKX", "SOL-USDT-SWAP"), + ("OKX", "DOGE-USDT-SWAP"), + ("OKX", "BNB-USDT-SWAP"), + }, + ) + def test_paper_quote_routes_keep_strict_freshness_and_session_sla(self): scope = build_release_consumer_acceptance_scope( self.release, @@ -141,14 +178,25 @@ def test_paper_quote_routes_keep_strict_freshness_and_session_sla(self): consumer_ids=FIVE_LIQUID_CONSUMER_IDS, ) quotes = [item for item in scope.products if item.feed is FeedType.QUOTE] - self.assertEqual(len(quotes), 10) + self.assertEqual(len(quotes), 20) self.assertTrue(all( item.requirement.event_recency_policy is None - and item.requirement.max_freshness_ms == 2_000 and item.requirement.max_session_liveness_ms == 45_000 and item.requirement.stale_policy is StalePolicy.BLOCK for item in quotes )) + self.assertEqual( + Counter(( + item.consumer_id, + item.requirement.consumer_grade.value, + item.requirement.max_freshness_ms, + ) for item in quotes), + { + ("alpha.binance.paper.stable", "ALPHA", 5_000): 5, + ("alpha.okx.paper.stable", "ALPHA", 5_000): 5, + ("trading-system.paper.stable", "EXECUTION", 2_000): 10, + }, + ) def test_execution_mark_and_l2_routes_are_exact_and_fail_closed(self): consumer = next( diff --git a/tests/test_phase105_execution_l2_status_matrix.py b/tests/test_phase105_execution_l2_status_matrix.py new file mode 100644 index 00000000..ca25f6df --- /dev/null +++ b/tests/test_phase105_execution_l2_status_matrix.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_deployment import StableAcquisitionPlan +from qdl_sdk.errors import DataLayerError +from scripts.phase105_execution_l2_status_matrix import ( + DEFAULT_ACQUISITION, + DEFAULT_CATALOG, + DEFAULT_EXECUTION_DEMAND, + DEFAULT_TRADING_MANIFEST, + execution_book_products, + ready_book_row, + replica_parity, + _read_one, +) + + +def _ready_row(*, source_id: str = "source-1", native_symbol: str = "BTCUSDT") -> dict[str, object]: + return { + "instrument_uid": "instrument-1", + "venue": "BINANCE", + "market": "USDM", + "native_symbol": native_symbol, + "feed": "BOOK_SNAPSHOT", + "source_policy_id": "crypto_primary_v2", + "source_id": source_id, + "depth": 100, + "typed_status": { + "quality": { + "state": "LIVE", + "complete": True, + "gap_open": False, + "execution_eligible": True, + } + }, + "snapshot": { + "sequence_verified": True, + "book_generation": 1, + "depth": 100, + "watermark_offset": 12, + "complete": True, + "gap_open": False, + "execution_eligible": True, + }, + } + + +class ExecutionL2StatusMatrixTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.catalog = StableSourceCatalog.load(DEFAULT_CATALOG) + cls.acquisition = StableAcquisitionPlan.load(DEFAULT_ACQUISITION, catalog=cls.catalog) + + def test_declared_execution_matrix_is_exactly_ten_physical_books(self) -> None: + products = execution_book_products( + catalog=self.catalog, + acquisition=self.acquisition, + execution_demand=DEFAULT_EXECUTION_DEMAND, + trading_manifest=DEFAULT_TRADING_MANIFEST, + ) + self.assertEqual(len(products), 10) + self.assertEqual( + {(item.venue, item.native_symbol) for item in products}, + { + ("BINANCE", symbol) + for symbol in ("BTCUSDT", "ETHUSDT", "SOLUSDT", "DOGEUSDT", "BNBUSDT") + } + | { + ("OKX", symbol) + for symbol in ( + "BTC-USDT-SWAP", + "ETH-USDT-SWAP", + "SOL-USDT-SWAP", + "DOGE-USDT-SWAP", + "BNB-USDT-SWAP", + ) + }, + ) + + def test_quality_matrix_fails_closed_for_gap_or_unverified_resync_view(self) -> None: + row = _ready_row() + self.assertTrue(ready_book_row(row)) + for field, value in ( + ( + "typed_status", + { + "quality": { + "state": "GAPPED", + "complete": False, + "gap_open": True, + "execution_eligible": False, + } + }, + ), + ("snapshot", {**row["snapshot"], "sequence_verified": False}), + ("snapshot", {**row["snapshot"], "book_generation": 0}), + ): + with self.subTest(field=field): + changed = dict(row) + changed[field] = value + self.assertFalse(ready_book_row(changed)) + + def test_replica_parity_rejects_cross_book_and_preserves_duplicate_ready_view(self) -> None: + primary = _ready_row() + duplicate_ready = _ready_row() + self.assertTrue(replica_parity(primary, duplicate_ready)) + cross_book = _ready_row(source_id="source-other", native_symbol="ETHUSDT") + self.assertFalse(replica_parity(primary, cross_book)) + + +class ExecutionL2StatusMatrixReadTests(unittest.IsolatedAsyncioTestCase): + async def test_status_transport_error_returns_compact_fail_closed_row(self) -> None: + class Client: + async def feed_status(self, _requirement): + raise DataLayerError("UNAVAILABLE", "query replica unavailable") + + async def close(self) -> None: + return None + + product = SimpleNamespace( + instrument_uid="book-uid", + instrument_id="BINANCE.USDM.PERPETUAL.BTCUSDT", + venue="BINANCE", + market="USDM", + native_symbol="BTCUSDT", + feed=SimpleNamespace(value="BOOK_SNAPSHOT"), + source_policy_id="crypto_primary_v2", + ) + with tempfile.TemporaryDirectory() as raw: + with patch( + "scripts.phase105_execution_l2_status_matrix.sdk_requirement", + return_value=object(), + ), patch( + "scripts.phase105_execution_l2_status_matrix._client", + return_value=Client(), + ): + row = await _read_one( + product, + label="primary", + base_url="https://query.example", + grpc_target="stream.example:8210", + identity=object(), + state_dir=Path(raw), + timeout_seconds=15.0, + ) + self.assertEqual(row["status_error"], { + "code": "UNAVAILABLE", + "detail": "query replica unavailable", + }) + self.assertIsNone(row["typed_status"]) + self.assertIsNone(row["snapshot"]) + self.assertFalse(ready_book_row(row)) + self.assertFalse(row["payload_recorded"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_phase105_fallback_acceptance.py b/tests/test_phase105_fallback_acceptance.py index c78b9f51..438b0553 100644 --- a/tests/test_phase105_fallback_acceptance.py +++ b/tests/test_phase105_fallback_acceptance.py @@ -91,6 +91,37 @@ def test_blocked_scope_never_appears_in_v1_probe_scope(self) -> None: self.assertFalse(blocked & {item.identity for item in self.probes}) self.assertTrue(any("alpha.okx" in consumer for consumer, _key in blocked)) + def test_execution_trade_fallback_is_blocked_without_v1_session_lineage(self) -> None: + """V1 last-trade age cannot certify the execution-grade 3s contract.""" + execution_products = [ + item for item in self.scope.products + if item.consumer_id == "trading-system.paper.stable" + and item.venue == "BINANCE" + and item.market == "USDM" + and item.feed.value == "TRADE" + ] + self.assertEqual( + {item.native_symbol for item in execution_products}, + {"BTCUSDT", "ETHUSDT", "SOLUSDT", "DOGEUSDT", "BNBUSDT"}, + ) + route_by_identity = { + (consumer.consumer_id, product.requirement_key): product + for consumer in self.release.consumers + for product in consumer.products + } + for item in execution_products: + route = route_by_identity[(item.consumer_id, requirement_key(item.requirement))] + self.assertEqual(route.route, "V2_PRIMARY") + self.assertEqual(route.fallback, "BLOCKED") + self.assertEqual(route.reason, "V1_EXECUTION_SESSION_LIVENESS_UNPROVEN") + self.assertFalse( + any(item.consumer_id == "trading-system.paper.stable" for item in self.probes) + ) + self.assertEqual( + {item.consumer_id for item in self.probes}, + {"monitoring.multivenue.stable", "alpha.binance.paper.stable"}, + ) + def test_trade_contract_is_checked_without_retaining_payload(self) -> None: trade = next(item for item in self.probes if item.feed == "TRADE") trade_result = validate_v1_fallback_payload(trade, { diff --git a/tests/test_phase105_handoff.py b/tests/test_phase105_handoff.py index 28b3f708..d54bdebb 100644 --- a/tests/test_phase105_handoff.py +++ b/tests/test_phase105_handoff.py @@ -9,12 +9,14 @@ from qdl.certification.phase105_handoff import ( ALL_KEY_SUBJECTS, + RECOVERY_ALL_KEY_SUBJECTS, V1_FALLBACK_COMMIT, V1_FALLBACK_VERSION, active_query_environment_commitment, active_runtime_binding, handoff_packet, load_dotenv, + prepare_c2_identity_recovery_environment, prepare_handoff_environment, public_handoff_overlay, render_dotenv, @@ -23,6 +25,9 @@ v1_image_attestation, ) from scripts.phase105_prepare_handoff_bundle import main as prepare_handoff_main +from scripts.phase105_prepare_c2_identity_recovery import ( + main as prepare_identity_recovery_main, +) class Phase105HandoffTests(unittest.TestCase): @@ -33,6 +38,9 @@ def setUp(self) -> None: "QDL_STABLE_JWT_KEYS_JSON": json.dumps({ "stable-trading-system-rs256-v1": "-----BEGIN PUBLIC KEY-----\\ntrading\\n-----END PUBLIC KEY-----", "stable-alpha-binance-rs256-v1": "-----BEGIN PUBLIC KEY-----\\nalpha\\n-----END PUBLIC KEY-----", + "stable-monitoring-rs256-v1": "-----BEGIN PUBLIC KEY-----\\nmonitoring-jwt/public.pem\\n-----END PUBLIC KEY-----\\n", + "stable-alpha-okx-rs256-v1": "-----BEGIN PUBLIC KEY-----\\nalpha-okx-jwt/public.pem\\n-----END PUBLIC KEY-----\\n", + "stable-reference-l2-rs256-v1": "-----BEGIN PUBLIC KEY-----\\nreference-l2-jwt/public.pem\\n-----END PUBLIC KEY-----\\n", }), } @@ -51,6 +59,22 @@ def _extension(self, root: Path) -> Path: (root / "client-ca-bundle.crt").write_text("server-ca\\nexternal-ca\\n", encoding="utf-8") return root + def _recovery_extension(self, root: Path) -> Path: + for relative in ( + "monitoring-jwt/public.pem", + "trading-system-jwt/public.pem", + "alpha-binance-jwt/public.pem", + "alpha-okx-jwt/public.pem", + ): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "-----BEGIN PUBLIC KEY-----\\n" + relative + "\\n-----END PUBLIC KEY-----\\n", + encoding="utf-8", + ) + (root / "client-ca-bundle.crt").write_text("server-ca\\nexternal-ca\\n", encoding="utf-8") + return root + def _final_bar_packet(self) -> dict[str, object]: runtime_dir = "/home/bobby/.local/state/qdl-v2/phase105c-test/runtime" rust_image = "sha256:" + "d" * 64 @@ -114,7 +138,7 @@ def test_load_dotenv_unquotes_standard_private_env_values(self) -> None: self.assertEqual(values["QDL_STABLE_JWT_KEYS_JSON"], '{"key":"value"}') self.assertEqual(values["QDL_STABLE_RUNTIME_DIR"], "/runtime") - def test_environment_has_exact_five_key_subject_bindings(self) -> None: + def test_environment_has_exact_legacy_key_subject_bindings(self) -> None: with tempfile.TemporaryDirectory() as raw: values = prepare_handoff_environment( self.base, @@ -126,6 +150,61 @@ def test_environment_has_exact_five_key_subject_bindings(self) -> None: self.assertEqual(json.loads(values["QDL_STABLE_JWT_KEY_SUBJECTS_JSON"]), ALL_KEY_SUBJECTS) self.assertNotIn("PRIVATE KEY", values["QDL_STABLE_JWT_KEYS_JSON"]) + def test_recovery_environment_adds_only_versioned_c2_keys(self) -> None: + with tempfile.TemporaryDirectory() as raw: + values = prepare_c2_identity_recovery_environment( + self.base, + extension_dir=self._recovery_extension(Path(raw)), + python_image="sha256:" + "b" * 64, + ) + keyring = json.loads(values["QDL_STABLE_JWT_KEYS_JSON"]) + self.assertEqual(set(keyring), set(RECOVERY_ALL_KEY_SUBJECTS)) + self.assertEqual( + keyring["stable-trading-system-rs256-v1"], + json.loads(self.base["QDL_STABLE_JWT_KEYS_JSON"])["stable-trading-system-rs256-v1"], + ) + self.assertEqual( + json.loads(values["QDL_STABLE_JWT_KEY_SUBJECTS_JSON"]), + RECOVERY_ALL_KEY_SUBJECTS, + ) + + def test_identity_recovery_packet_is_public_only_and_additive(self) -> None: + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + base_path = root / "stable.env" + v1_path = root / "v1.json" + output = root / "output" + base_path.write_text(render_dotenv(self.base), encoding="utf-8") + v1_path.write_text(json.dumps({ + "schema": "qdl.phase105.v1-fallback-provenance.v1", + "status": "PASS", + "image_id": "sha256:" + "a" * 64, + "source_commit": V1_FALLBACK_COMMIT, + "source_tree": "b" * 40, + "dockerfile_sha256": "c" * 64, + "version": V1_FALLBACK_VERSION, + }), encoding="utf-8") + stdout = io.StringIO() + with redirect_stdout(stdout): + status = prepare_identity_recovery_main([ + "--base-env", str(base_path), + "--extension-dir", str(self._recovery_extension(root / "extension")), + "--reader-image", "sha256:" + "d" * 64, + "--v1-provenance", str(v1_path), + "--output-dir", str(output), + "--apply", + "--confirm", "PREPARE_QDL_PHASE105C_IDENTITY_RECOVERY", + ]) + self.assertEqual(status, 0) + self.assertEqual((output / "identity-recovery-public.env").stat().st_mode & 0o777, 0o600) + packet = json.loads((output / "identity-recovery-packet.json").read_text()) + self.assertEqual(packet["schema"], "qdl.phase105c.identity-recovery.v1") + self.assertEqual(packet["retained_key_ids"], sorted(ALL_KEY_SUBJECTS)) + self.assertEqual(set(packet["recovery_key_ids"]), set(RECOVERY_ALL_KEY_SUBJECTS) - set(ALL_KEY_SUBJECTS)) + encoded = (output / "identity-recovery-public.env").read_text() + self.assertNotIn("PRIVATE KEY", encoded) + self.assertNotIn("PRIVATE KEY", stdout.getvalue()) + def test_active_runtime_packet_only_overlays_allowlisted_selectors(self) -> None: with tempfile.TemporaryDirectory() as raw: base = { @@ -366,7 +445,11 @@ def test_active_query_commitment_keeps_secret_values_out_of_handoff_output(self) }) self.assertNotIn("unchanged-secret", json.dumps(overlay, sort_keys=True)) self.assertEqual(set(verified["verified_keys"]), set(current)) - current["QDL_DATA_JWT_KEYS_JSON"] = base["QDL_STABLE_JWT_KEYS_JSON"] + current["QDL_DATA_JWT_KEYS_JSON"] = json.dumps({ + key: value + for key, value in json.loads(base["QDL_STABLE_JWT_KEYS_JSON"]).items() + if key != "stable-reference-l2-rs256-v1" + }) with self.assertRaisesRegex(ValueError, "JWT keyring mismatches public overlay"): active_query_environment_commitment( base, current, runtime, json.loads(expected["QDL_STABLE_JWT_KEYS_JSON"]) @@ -498,7 +581,24 @@ def test_compose_overrides_remain_bounded(self) -> None: self.assertNotIn("data_layer_service:", c2) self.assertIn("build: !reset null", v1) self.assertIn("volumes: !override", v1) - self.assertNotIn(":/app\n", v1) + self.assertNotIn(":/app\\n", v1) + + def test_base_readers_use_additive_client_ca_with_one_ca_fallback(self) -> None: + root = Path(__file__).resolve().parents[1] + stable = (root / "docker-compose.v2-stable.yml").read_text(encoding="utf-8") + self.assertEqual(stable.count("QDL_STABLE_TLS_CLIENT_CA_FILE:"), 4) + self.assertEqual(stable.count("/stable-certs/query/client-ca-bundle.crt"), 4) + self.assertEqual(stable.count("/stable-certs/stream/client-ca-bundle.crt"), 4) + self.assertIn( + "test -f /stable-certs/query/client-ca-bundle.crt || cp " + "/stable-certs/query/ca.crt /stable-certs/query/client-ca-bundle.crt", + stable, + ) + self.assertIn( + "test -f /stable-certs/stream/client-ca-bundle.crt || cp " + "/stable-certs/stream/ca.crt /stable-certs/stream/client-ca-bundle.crt", + stable, + ) if __name__ == "__main__": diff --git a/tests/test_phase105_identity_acceptance.py b/tests/test_phase105_identity_acceptance.py index 6e1815c7..db29fbd7 100644 --- a/tests/test_phase105_identity_acceptance.py +++ b/tests/test_phase105_identity_acceptance.py @@ -4,27 +4,165 @@ import tempfile import unittest import asyncio +import httpx +from dataclasses import replace from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch -from qdl.certification.phase103_consumer_acceptance import AcceptanceProduct +from qdl.certification.phase103_consumer_acceptance import AcceptanceProduct, DeliveryClass from qdl.query import DataRequirement, FeedType, RecoveryPolicy -from qdl_sdk import Grade +from qdl_sdk import ( + DataRequirement as SdkDataRequirement, + Feed, + FeedStatusResponse, + Grade, + WarmupSpecification as SdkWarmupSpecification, +) +from scripts.phase103_consumer_receipt_acceptance import C2StatusEvidenceError from scripts.phase105_consumer_v2_identity_acceptance import ( + C2ProductAcceptanceError, + C2ClosingBatchError, + _C2ConsumerRequestPacer, + _PacedQueryTransport, + _PacedStreamTransport, IDENTITY_PREFIXES, _authority, + _closing_batch_revalidation, + _closing_batches, + _closing_requirement, _c2_grpc_targets, _consumer_ids, _identity_files, _identity_files_for_consumers, _route_summary, _reference_batch_concurrency, + _reference_transport_timeout_seconds, _run_consumer_groups, + _paced_client_factory, + _wait_for_minimum_observation, _v1_base_url, parser, ) class Phase105IdentityAcceptanceTests(unittest.TestCase): + def test_closing_batches_isolate_hot_feeds_without_losing_scope(self) -> None: + products = tuple(SimpleNamespace(identity=(venue, symbol, feed), feed=Feed(feed)) + for venue in ("BINANCE", "OKX") + for symbol in ("BTC", "ETH", "SOL", "DOGE", "BNB") + for feed in ("TRADE", "BAR", "QUOTE", "BOOK_SNAPSHOT", "BOOK_DELTA")) + for limit in (1, 5, 50): + batches = tuple(_closing_batches(products, limit)) + actual = [item.identity for batch in batches for item in batch] + self.assertCountEqual(actual, [item.identity for item in products]) + self.assertEqual(len(actual), len(set(actual))) + for batch in batches: + self.assertEqual(len({item.feed for item in batch}), 1) + bound = limit if batch[0].feed is Feed.BAR else min(limit, 8) + self.assertLessEqual(len(batch), bound) + self.assertEqual(tuple(_closing_batches((), 50)), ()) + + def test_typed_c2_product_failure_keeps_status_without_market_payload(self) -> None: + status = FeedStatusResponse.model_validate({ + "schema": "qdl.feed-status.v2", + "instrument_uid": "book-uid", + "feed": "BOOK_SNAPSHOT", + "quality": { + "state": "GAPPED", + "freshness_ms": 12, + "event_recency_state": "LIVE", + "provider_session_state": "NOT_APPLICABLE", + "provider_session_liveness_ms": None, + "gap_open": True, + "complete": False, + "execution_eligible": False, + "policy_id": "crypto_liquid_v2", + "flags": ["SEQUENCE_GAP"], + }, + }) + error = C2StatusEvidenceError("DATA_STALE", "strict book rejected", status=status) + error.replica = "secondary" + product = SimpleNamespace( + consumer_id="trading-system.paper.stable", + instrument_id="OKX.SWAP.PERPETUAL.SOL-USDT", + feed=Feed.BOOK_SNAPSHOT, + interval=None, + evidence=lambda: {"instrument_uid": "book-uid", "feed": "BOOK_SNAPSHOT"}, + ) + failure = C2ProductAcceptanceError(product, error) + self.assertEqual(failure.evidence["replica"], "secondary") + self.assertEqual(failure.evidence["typed_status"]["quality"]["state"], "GAPPED") + self.assertFalse(failure.evidence["payload_recorded"]) + self.assertNotIn("levels", repr(failure.evidence)) + + def test_closing_bar_requirement_keeps_policy_and_reduces_only_history_rows(self) -> None: + requirement = SdkDataRequirement( + instrument_uid="bar-uid", + feed=Feed.BAR, + consumer_grade=Grade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="15m", + warmup_limit=700, + max_freshness_ms=5_000, + max_session_liveness_ms=8_000, + require_full_coverage=True, + require_final_bars=True, + warmup=SdkWarmupSpecification( + rows=700, + deadline_ms=9_000, + max_cache_age_ms=7_000, + ), + ) + product = SimpleNamespace() + with patch( + "scripts.phase105_consumer_v2_identity_acceptance.sdk_requirement", + return_value=requirement, + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance._c2_requirement", + side_effect=lambda value: value, + ): + closing = _closing_requirement(product) + self.assertEqual(requirement.warmup_limit, 700) + self.assertEqual(requirement.warmup.rows, 700) + self.assertEqual(closing.warmup_limit, 2) + self.assertEqual(closing.warmup.rows, 2) + self.assertEqual(closing.max_freshness_ms, requirement.max_freshness_ms) + self.assertEqual(closing.max_session_liveness_ms, requirement.max_session_liveness_ms) + self.assertEqual(closing.require_full_coverage, requirement.require_full_coverage) + self.assertEqual(closing.require_final_bars, requirement.require_final_bars) + self.assertEqual(closing.warmup.deadline_ms, requirement.warmup.deadline_ms) + self.assertEqual(closing.warmup.max_cache_age_ms, requirement.warmup.max_cache_age_ms) + + limit_only = replace(requirement, warmup=None) + with patch( + "scripts.phase105_consumer_v2_identity_acceptance.sdk_requirement", + return_value=limit_only, + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance._c2_requirement", + side_effect=lambda value: value, + ): + limit_only_closing = _closing_requirement(product) + self.assertIsNone(limit_only_closing.warmup) + self.assertEqual(limit_only_closing.warmup_limit, 2) + + def test_paced_client_factory_wraps_both_c2_transports(self) -> None: + client = SimpleNamespace(query_transport=object(), stream_transport=object()) + with patch( + "scripts.phase105_consumer_v2_identity_acceptance._client", + return_value=client, + ): + result = _paced_client_factory(_C2ConsumerRequestPacer(8))( + object(), + base_url="https://query.example", + grpc_target="stream.example:8210", + cursor_path=Path("/tmp/cursor.json"), + timeout_seconds=15.0, + ) + self.assertIs(result, client) + self.assertIsInstance(result.query_transport, _PacedQueryTransport) + self.assertIsInstance(result.stream_transport, _PacedStreamTransport) + def test_identity_prefixes_are_exactly_the_four_governed_paper_consumers(self) -> None: self.assertEqual(set(IDENTITY_PREFIXES), { "monitoring.multivenue.stable", @@ -156,6 +294,27 @@ def test_reference_batches_are_capped_without_throttling_product_reads(self) -> with self.assertRaisesRegex(ValueError, "positive"): _reference_batch_concurrency(0) + def test_reference_transport_uses_declared_deadline_not_generic_read_timeout(self) -> None: + products = ( + SimpleNamespace(sdk_requirement=SimpleNamespace(deadline_ms=60_000)), + SimpleNamespace(sdk_requirement=SimpleNamespace(deadline_ms=20_000)), + ) + self.assertEqual( + _reference_transport_timeout_seconds( + products, generic_timeout_seconds=15.0 + ), + 75.0, + ) + self.assertEqual( + _reference_transport_timeout_seconds( + (SimpleNamespace(sdk_requirement=SimpleNamespace(deadline_ms=5_000)),), + generic_timeout_seconds=15.0, + ), + 30.0, + ) + with self.assertRaisesRegex(ValueError, "at least one"): + _reference_transport_timeout_seconds((), generic_timeout_seconds=15.0) + def test_route_summary_counts_declared_v1_and_blocked_without_claiming_transition(self) -> None: requirement = DataRequirement( instrument_uid="a953e16e-7138-5562-b5e8-c337a44d0b65", @@ -203,6 +362,346 @@ class Release: class Phase105ConcurrentConsumerGroupTests(unittest.IsolatedAsyncioTestCase): + async def test_c2_pacer_aligns_then_spaces_requests_below_manifest_quota(self) -> None: + clock = {"value": 100.0} + sleeps: list[float] = [] + + async def fake_sleep(duration: float) -> None: + sleeps.append(duration) + clock["value"] += duration + + pacer = _C2ConsumerRequestPacer( + 4, + safety_fraction=0.5, + clock=lambda: clock["value"], + sleep=fake_sleep, + ) + self.assertAlmostEqual(await pacer.wait_for_clean_window(), 20.05) + await pacer.acquire() + await pacer.acquire() + await pacer.acquire() + self.assertEqual(len(sleeps), 3) + for actual, expected in zip(sleeps, (20.05, 30.0, 30.0), strict=True): + self.assertAlmostEqual(actual, expected) + self.assertEqual(pacer.evidence()["c2_safe_requests_per_minute"], 2) + self.assertEqual(pacer.evidence()["c2_request_count"], 3) + + async def test_stream_open_and_rest_read_share_one_identity_budget(self) -> None: + clock = {"value": 0.0} + sleeps: list[float] = [] + + async def fake_sleep(duration: float) -> None: + sleeps.append(duration) + clock["value"] += duration + + class QueryDelegate: + def __init__(self) -> None: + self.calls = 0 + + async def snapshot(self, *args, **kwargs): + del args, kwargs + self.calls += 1 + return {"snapshot": True} + + async def close(self) -> None: + return None + + class StreamDelegate: + def __init__(self) -> None: + self.calls = 0 + + async def subscribe(self, *args, **kwargs): + del args, kwargs + self.calls += 1 + yield "frame" + + async def close(self) -> None: + return None + + pacer = _C2ConsumerRequestPacer( + 4, + safety_fraction=0.5, + clock=lambda: clock["value"], + sleep=fake_sleep, + ) + query_delegate = QueryDelegate() + stream_delegate = StreamDelegate() + query = _PacedQueryTransport(query_delegate, pacer) + stream = _PacedStreamTransport(stream_delegate, pacer) + + self.assertEqual(await query.snapshot(object()), {"snapshot": True}) + self.assertEqual(await anext(stream.subscribe(object())), "frame") + self.assertEqual(query_delegate.calls, 1) + self.assertEqual(stream_delegate.calls, 1) + self.assertEqual(sleeps, [30.0]) + self.assertEqual(pacer.evidence()["c2_request_count"], 2) + + async def test_stream_open_failure_remains_fail_closed(self) -> None: + class StreamDelegate: + async def subscribe(self, *args, **kwargs): + del args, kwargs + raise RuntimeError("stream transport rejected subscription") + yield None + + async def close(self) -> None: + return None + + pacer = _C2ConsumerRequestPacer(8) + stream = _PacedStreamTransport(StreamDelegate(), pacer) + with self.assertRaisesRegex(RuntimeError, "rejected subscription"): + await anext(stream.subscribe(object())) + self.assertEqual(pacer.evidence()["c2_request_count"], 1) + + async def test_distinct_identity_stream_pacers_do_not_share_a_lock(self) -> None: + opened: list[str] = [] + + class StreamDelegate: + def __init__(self, label: str) -> None: + self.label = label + + async def subscribe(self, *args, **kwargs): + del args, kwargs + opened.append(self.label) + yield self.label + + async def close(self) -> None: + return None + + first = _PacedStreamTransport(StreamDelegate("first"), _C2ConsumerRequestPacer(8)) + second = _PacedStreamTransport(StreamDelegate("second"), _C2ConsumerRequestPacer(8)) + values = await asyncio.gather( + anext(first.subscribe(object())), + anext(second.subscribe(object())), + ) + self.assertEqual(set(values), {"first", "second"}) + self.assertEqual(set(opened), {"first", "second"}) + + async def test_closing_batch_revalidates_every_product_on_both_replicas(self) -> None: + class Product: + def __init__(self, name: str) -> None: + self.consumer_id = "alpha.binance.paper.stable" + self.instrument_uid = f"uid-{name}" + self.instrument_id = f"BINANCE.USDM.PERPETUAL.{name}-USDT" + self.feed = SimpleNamespace(value="TRADE") + self.interval = None + self.source_policy_id = "crypto_primary_v2" + self.delivery = DeliveryClass.DURABLE + self.requirement = object() + self.identity = ( + self.consumer_id, self.instrument_uid, "TRADE", "", + self.source_policy_id, + ) + + def evidence(self) -> dict[str, object]: + return { + "consumer_id": self.consumer_id, + "instrument_uid": self.instrument_uid, + "feed": "TRADE", + "interval": None, + "source_policy_id": self.source_policy_id, + } + + products = (Product("BTC"), Product("ETH")) + clients = [] + + class Client: + def __init__(self, label: str) -> None: + self.label = label + self.calls: list[tuple[object, ...]] = [] + + async def warmup_batch(self, requirements, *, require_all: bool): + if not require_all: + raise AssertionError("closing batch must require every product") + self.calls.append(tuple(requirements)) + return SimpleNamespace( + partial=False, + results=[ + SimpleNamespace(data=SimpleNamespace(data=[SimpleNamespace()])) + for _ in requirements + ], + ) + + async def close(self) -> None: + return None + + def factory(identity, *, base_url, grpc_target, cursor_path, timeout_seconds): + del identity, grpc_target, cursor_path, timeout_seconds + client = Client(base_url) + clients.append(client) + return client + + with patch( + "scripts.phase105_consumer_v2_identity_acceptance._closing_requirement", + side_effect=lambda product: product.requirement, + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance.validate_product_view", + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance.validate_replica_views", + return_value=("a" * 64, "b" * 64), + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance.compact_view_quality", + return_value={"state": "LIVE"}, + ): + evidence = await _closing_batch_revalidation( + products, + identity=object(), + primary_url="https://primary", + secondary_url="https://secondary", + grpc_target="stream:8210", + state_dir=Path("/tmp/phase105-closing"), + timeout_seconds=15.0, + max_batch_items=1, + client_factory=factory, + ) + self.assertEqual(len(evidence), 2) + self.assertEqual({item["closing_read"] for item in evidence}, {"BATCH_V2_PRIMARY"}) + self.assertEqual(len(clients), 2) + self.assertEqual([len(call) for client in clients for call in client.calls], [1, 1, 1, 1]) + + async def test_closing_batch_rejects_partial_cardinality(self) -> None: + product = SimpleNamespace( + consumer_id="alpha.binance.paper.stable", + instrument_uid="uid-btc", + instrument_id="BINANCE.USDM.PERPETUAL.BTC-USDT", + feed=SimpleNamespace(value="TRADE"), + interval=None, + source_policy_id="crypto_primary_v2", + delivery=DeliveryClass.DURABLE, + requirement=object(), + identity=("alpha.binance.paper.stable", "uid-btc", "TRADE", "", "crypto_primary_v2"), + evidence=lambda: { + "consumer_id": "alpha.binance.paper.stable", + "instrument_uid": "uid-btc", + "feed": "TRADE", + "interval": None, + "source_policy_id": "crypto_primary_v2", + }, + ) + + class Client: + async def warmup_batch(self, requirements, *, require_all: bool): + del requirements, require_all + return SimpleNamespace(partial=False, results=[]) + + async def close(self) -> None: + return None + + def factory(*args, **kwargs): + del args, kwargs + return Client() + + with patch( + "scripts.phase105_consumer_v2_identity_acceptance._closing_requirement", + return_value=product.requirement, + ): + with self.assertRaisesRegex(AssertionError, "cardinality"): + await _closing_batch_revalidation( + (product,), + identity=object(), + primary_url="https://primary", + secondary_url="https://secondary", + grpc_target="stream:8210", + state_dir=Path("/tmp/phase105-closing"), + timeout_seconds=15.0, + max_batch_items=50, + client_factory=factory, + ) + + async def test_closing_batch_timeout_has_bounded_typed_status_evidence(self) -> None: + product = SimpleNamespace( + consumer_id="alpha.binance.paper.stable", + instrument_uid="uid-btc", + instrument_id="BINANCE.USDM.PERPETUAL.BTC-USDT", + feed=Feed.TRADE, + interval=None, + source_policy_id="crypto_primary_v2", + delivery=DeliveryClass.DURABLE, + requirement=object(), + identity=("alpha.binance.paper.stable", "uid-btc", "TRADE", "", "crypto_primary_v2"), + evidence=lambda: { + "consumer_id": "alpha.binance.paper.stable", + "instrument_uid": "uid-btc", + "feed": "TRADE", + "interval": None, + "source_policy_id": "crypto_primary_v2", + }, + ) + status = FeedStatusResponse.model_validate({ + "schema": "qdl.feed-status.v2", + "instrument_uid": "uid-btc", + "feed": "TRADE", + "quality": { + "state": "LIVE", + "freshness_ms": 7, + "event_recency_state": "LIVE", + "provider_session_state": "LIVE", + "provider_session_liveness_ms": 3, + "gap_open": False, + "complete": True, + "execution_eligible": True, + "policy_id": "crypto_primary_v2", + "flags": [], + }, + }) + + class Client: + async def warmup_batch(self, requirements, *, require_all: bool): + del requirements, require_all + raise httpx.ReadTimeout("closing request timed out") + + async def feed_status(self, requirement): + del requirement + return status + + async def close(self) -> None: + return None + + def factory(*args, **kwargs): + del args, kwargs + return Client() + + with patch( + "scripts.phase105_consumer_v2_identity_acceptance._closing_requirement", + return_value=product.requirement, + ): + with self.assertRaises(C2ClosingBatchError) as raised: + await _closing_batch_revalidation( + (product,), + identity=object(), + primary_url="https://primary", + secondary_url="https://secondary", + grpc_target="stream:8210", + state_dir=Path("/tmp/phase105-closing"), + timeout_seconds=15.0, + max_batch_items=50, + client_factory=factory, + ) + evidence = raised.exception.evidence + self.assertEqual(evidence["transport_error"], "ReadTimeout") + self.assertEqual(evidence["batch_size"], 1) + self.assertEqual(evidence["typed_status"][0]["quality"]["quality"]["state"], "LIVE") + self.assertFalse(evidence["payload_recorded"]) + self.assertNotIn("price", repr(evidence)) + + async def test_observation_waits_until_the_declared_floor(self) -> None: + clock = {"value": 100.0} + + async def fake_sleep(duration: float) -> None: + clock["value"] += duration + + with patch( + "scripts.phase105_consumer_v2_identity_acceptance.time.monotonic", + side_effect=lambda: clock["value"], + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance.asyncio.sleep", + side_effect=fake_sleep, + ): + elapsed = await _wait_for_minimum_observation( + started_monotonic=100.0, + observation_seconds=300.0, + ) + self.assertEqual(elapsed, 300.0) + async def test_groups_start_in_declared_order_and_collect_in_that_order(self) -> None: started: list[str] = [] release = asyncio.Event() diff --git a/tests/test_phase105_native_basis_lane.py b/tests/test_phase105_native_basis_lane.py new file mode 100644 index 00000000..5e83dfbc --- /dev/null +++ b/tests/test_phase105_native_basis_lane.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import asyncio +from pathlib import Path +from types import SimpleNamespace +import time +import unittest +from unittest.mock import AsyncMock, patch + +from qdl.query import FeedType +from qdl_sdk.reference import BasisSeries +from scripts.phase105_consumer_v2_identity_acceptance import ( + _closing_revalidate_consumer, _reference_batch_for_c2, +) +from qdl.certification.phase103_consumer_acceptance import DeliveryClass +from scripts.phasec36_reference_l2_consumer_acceptance import _reference_batch_until_terminal + + +class Phase105NativeBasisLaneTests(unittest.IsolatedAsyncioTestCase): + async def test_closing_references_share_absolute_bound_not_cache_timeout(self): + products = (SimpleNamespace(delivery=DeliveryClass.DURABLE), + SimpleNamespace(delivery=DeliveryClass.ON_DEMAND)) + prefix = "scripts.phase105_consumer_v2_identity_acceptance." + for venue in ("BINANCE", "OKX"): + with self.subTest(venue=venue), patch( + prefix + "_closing_batch_revalidation", new_callable=AsyncMock, + return_value=["stream"], + ) as stream, patch( + prefix + "_certify_references", new_callable=AsyncMock, + return_value=["reference"], + ) as reference: + deadline = time.monotonic() + 120 + result = await _closing_revalidate_consumer( + venue, products, identity=object(), primary_url="https://primary", + secondary_url="https://secondary", grpc_target="stream:8210", + state_dir=Path("/tmp/not-created"), timeout_seconds=15, + deadline_monotonic=deadline, max_batch_items=50, + reference_semaphore=asyncio.Semaphore(2), + native_basis_semaphore=asyncio.Semaphore(1), client_factory=object(), + ) + self.assertEqual(result, ["stream", "reference"]) + self.assertEqual(reference.await_args.kwargs["deadline_monotonic"], deadline) + self.assertEqual(reference.await_args.kwargs["timeout_seconds"], 15) + self.assertEqual(stream.await_args.kwargs["timeout_seconds"], 15) + + async def test_native_cooldown_fits_existing_closing_bound_but_never_extends_it(self): + product = self._product(venue="BINANCE", native=True) + now = 100.0 + + async def sleep(seconds): + nonlocal now + now += seconds + + response = SimpleNamespace(partial=True, success_count=0, error_count=1, results=( + SimpleNamespace(status="SOURCE_UNAVAILABLE", data=None, problem=SimpleNamespace( + code="SOURCE_UNAVAILABLE", retryable=True, retry_after_ms=60_000)),)) + success = SimpleNamespace(partial=False) + client = SimpleNamespace(reference_batch=AsyncMock(side_effect=[response, success])) + result = await _reference_batch_until_terminal( + client, (product,), deadline_monotonic=220, clock=lambda: now, sleep=sleep, + ) + self.assertEqual(result, (success, 2, 60_000)) + self.assertEqual(now, 160) + for remaining in (15, 60): + with self.subTest(remaining=remaining): + client = SimpleNamespace(reference_batch=AsyncMock(return_value=response)) + with self.assertRaisesRegex(AssertionError, "retry_after_ms=60000"): + await _reference_batch_until_terminal( + client, (product,), deadline_monotonic=now+remaining, + clock=lambda: now, sleep=sleep, + ) + self.assertEqual(client.reference_batch.await_count, 1) + + @staticmethod + def _product(*, venue: str, native: bool): + return SimpleNamespace( + venue=venue, + requirement=SimpleNamespace(feed=FeedType.BASIS), + sdk_requirement=SimpleNamespace( + basis_series=BasisSeries.NATIVE if native else BasisSeries.CONTINUOUS + ), + ) + + async def test_native_basis_batches_share_one_lane_across_replicas(self): + product = self._product(venue="BINANCE", native=True) + active = 0 + maximum = 0 + + async def terminal(*_args, **_kwargs): + nonlocal active, maximum + active += 1 + maximum = max(maximum, active) + await asyncio.sleep(0.01) + active -= 1 + return SimpleNamespace(), 1, 0 + + native_lane = asyncio.Semaphore(1) + with patch( + "scripts.phase105_consumer_v2_identity_acceptance._reference_batch_until_terminal", + side_effect=terminal, + ): + results = await asyncio.gather( + _reference_batch_for_c2( + object(), (product,), deadline_monotonic=time.monotonic() + 1, + semaphore=asyncio.Semaphore(2), native_basis_semaphore=native_lane, + ), + _reference_batch_for_c2( + object(), (product,), deadline_monotonic=time.monotonic() + 1, + semaphore=asyncio.Semaphore(2), native_basis_semaphore=native_lane, + ), + ) + + self.assertEqual(maximum, 1) + self.assertEqual([item[1:] for item in results], [(1, 0), (1, 0)]) + + async def test_non_native_reference_does_not_wait_for_native_lane(self): + product = self._product(venue="OKX", native=False) + native_lane = asyncio.Semaphore(1) + await native_lane.acquire() + + async def terminal(*_args, **_kwargs): + return SimpleNamespace(), 1, 0 + + try: + with patch( + "scripts.phase105_consumer_v2_identity_acceptance._reference_batch_until_terminal", + side_effect=terminal, + ): + result = await asyncio.wait_for( + _reference_batch_for_c2( + object(), (product,), deadline_monotonic=time.monotonic() + 1, + semaphore=asyncio.Semaphore(1), native_basis_semaphore=native_lane, + ), + timeout=0.1, + ) + finally: + native_lane.release() + + self.assertEqual(result[1:], (1, 0)) diff --git a/tests/test_phase105_release_observations.py b/tests/test_phase105_release_observations.py index 36c8dc48..f9243dfc 100644 --- a/tests/test_phase105_release_observations.py +++ b/tests/test_phase105_release_observations.py @@ -1,6 +1,9 @@ from __future__ import annotations from pathlib import Path +from copy import deepcopy +import hashlib +import json from types import SimpleNamespace import unittest @@ -20,6 +23,65 @@ class Phase105ReleaseObservationTests(unittest.TestCase): + def current_reads(self, acceptance): + return { + "schema": "qdl.release.current-reads.v1", "status": "PASS", + "release_route_plan_sha256": self.plan.digest, + "acceptance_sha256": hashlib.sha256(json.dumps(acceptance, sort_keys=True, + separators=(",", ":")).encode()).hexdigest(), + "release_capture": {**acceptance["release_capture"], "captured_at_ms": self.captured_at_ms+1000}, + "products": deepcopy(acceptance["products"]), + } + + def test_current_reads_bind_full_scope_and_do_not_rewrite_original_c2(self): + acceptance = self.acceptance() + original = deepcopy(acceptance) + reads = self.current_reads(acceptance) + for item in reads["products"]: + for replica in ("primary", "secondary"): + item["release_quality"][replica]["source_age_ms"] = 7 + bundle = build_release_observation_bundle(self.plan, acceptance, current_reads=reads) + self.assertEqual(acceptance, original) + self.assertEqual(bundle["captured_at_ms"], self.captured_at_ms+1000) + self.assertIn("current_reads_sha256", bundle) + values = parse_release_observation_bundle(self.plan, bundle, now_ms=self.captured_at_ms+1000) + self.assertTrue(all(v.v2_source_age_ms == 7 for v in values if v.route == "V2_PRIMARY")) + with self.assertRaisesRegex(ValueError, "stale"): + parse_release_observation_bundle(self.plan, bundle, + now_ms=self.captured_at_ms+1001+MAX_OBSERVATION_AGE_MS) + + def test_current_read_wrong_c2_scope_identity_or_partial_cannot_certify(self): + acceptance = self.acceptance() + for case in ("digest", "plan", "partial", "duplicate", "cross_identity", "status", "old"): + reads = self.current_reads(acceptance) + if case == "digest": + reads["acceptance_sha256"] = "0"*64 + elif case == "plan": + reads["release_route_plan_sha256"] = "0"*64 + elif case == "partial": + reads["products"].pop() + elif case == "duplicate": + reads["products"].append(reads["products"][0]) + elif case == "cross_identity": + reads["products"][0]["instrument_uid"] = "other" + elif case == "old": + reads["release_capture"]["captured_at_ms"] = self.captured_at_ms-1 + else: + reads["status"] = "FAIL" + with self.subTest(case=case), self.assertRaises(ValueError): + build_release_observation_bundle(self.plan, acceptance, current_reads=reads) + + def test_closing_quality_supersedes_opening_but_identity_stays_strict(self): + acceptance = self.acceptance() + product = acceptance["products"][0] + product["closing_v2_read"] = deepcopy(product) + product["closing_v2_read"]["release_quality"]["primary"]["source_age_ms"] = 99 + bundle = build_release_observation_bundle(self.plan, acceptance) + self.assertEqual(next(v for v in bundle["observations"] if v["route"] == "V2_PRIMARY")["v2_source_age_ms"], 99) + product["closing_v2_read"]["instrument_uid"] = "other" + with self.assertRaises(ValueError): + build_release_observation_bundle(self.plan, acceptance) + def setUp(self) -> None: self.plan = StableReleaseRoutePlan.load(ROUTE_PATH, manifest_root=ROOT) self.captured_at_ms = 1_000_000 @@ -93,13 +155,13 @@ def test_full_manifest_bundle_marks_v1_only_routes_as_excluded(self) -> None: values = parse_release_observation_bundle( self.plan, bundle, now_ms=self.captured_at_ms ) - self.assertEqual(len(values), 153) + self.assertEqual(len(values), 303) v1_values = [item for item in values if item.route == "V1_PRIMARY"] self.assertEqual(len(v1_values), 4) self.assertTrue(all(item.reason == "VN_REAL_PROVIDER_GATE_UNEXERCISED" for item in v1_values)) self.assertTrue(all(item.v1_source_age_ms is None for item in v1_values)) v2_values = [item for item in values if item.route == "V2_PRIMARY"] - self.assertEqual(len(v2_values), 149) + self.assertEqual(len(v2_values), 299) self.assertTrue(all(item.v2_source_age_ms == 11 for item in v2_values)) deliveries = self.expected_deliveries() self.assertEqual( @@ -175,6 +237,16 @@ def test_bundle_rejects_missing_route_bad_resume_and_stale_capture(self) -> None now_ms=self.captured_at_ms + MAX_OBSERVATION_AGE_MS + 1, ) + def test_quiet_bar_observation_requires_two_final_bar_sessions(self) -> None: + from qdl.certification.phase105_release_observations import _durable_no_cursor_lag + value = {"feed": "BAR", "stream_handoff": "CURRENT_FINAL_BAR_OBSERVED_NO_CURSOR", + "stream_no_event_sessions": ["CURRENT_FINAL_BAR", "CURRENT_FINAL_BAR"]} + self.assertEqual(_durable_no_cursor_lag(value, index=0), 0) + for fields in ({"feed": "TRADE"}, {"stream_no_event_sessions": []}, + {"stream_no_event_sessions": ["CURRENT_FINAL_BAR", "UNKNOWN"]}): + with self.subTest(fields=fields), self.assertRaisesRegex(ValueError, "both current sessions"): + _durable_no_cursor_lag(dict(value, **fields), index=0) + def test_bundle_accepts_proven_live_durable_handoff_without_cursor(self) -> None: acceptance = self.acceptance() products = acceptance["products"] diff --git a/tests/test_phase105_stable_release.py b/tests/test_phase105_stable_release.py index 045b7785..692f8e47 100644 --- a/tests/test_phase105_stable_release.py +++ b/tests/test_phase105_stable_release.py @@ -37,7 +37,7 @@ def test_release_scope_is_per_requirement_and_fails_closed(self): "2b0dcf74454c9f87c352d3c47389955aeb955804", ) self.assertEqual(len(plan.consumers), 5) - self.assertEqual(len(plan.products()), 153) + self.assertEqual(len(plan.products()), 303) self.assertEqual( { consumer.consumer_id: len(consumer.products) @@ -46,8 +46,8 @@ def test_release_scope_is_per_requirement_and_fails_closed(self): { "monitoring.multivenue.stable": 5, "trading-system.paper.stable": 61, - "alpha.binance.paper.stable": 50, - "alpha.okx.paper.stable": 35, + "alpha.binance.paper.stable": 125, + "alpha.okx.paper.stable": 110, "alpha.vn.paper.stable": 2, }, ) @@ -110,7 +110,7 @@ def test_artifact_manifest_and_route_mutations_are_rejected(self): for consumer in incompatible_v1["consumers"] if consumer["consumer_id"] == "alpha.binance.paper.stable" for item in consumer["products"] - if ":BAR:1m:" in item["requirement_key"] + if ":QUOTE::" in item["requirement_key"] ) product["fallback"] = "V1" product["reason"] = None diff --git a/tests/test_phase10_universal_warmup.py b/tests/test_phase10_universal_warmup.py index a66aab48..4c0c8503 100644 --- a/tests/test_phase10_universal_warmup.py +++ b/tests/test_phase10_universal_warmup.py @@ -408,6 +408,34 @@ def test_resolver_rejects_ambiguous_row_and_range_coalescing(self): class WarmupExecutorTests(unittest.IsolatedAsyncioTestCase): + async def test_local_cache_has_bounded_concurrency_without_provider_token_wait(self): + sleeps = [] + running = 0 + peak = 0 + + async def sleep(delay): + sleeps.append(delay) + + async def work(value): + nonlocal running, peak + running += 1 + peak = max(peak, running) + await asyncio.sleep(0) + running -= 1 + return value + + executor = BoundedWarmupExecutor(sleep=sleep) + result = await executor.execute( + range(100), work=work, identity=lambda value: value, + provider=lambda _: "LOCAL_CANONICAL_CACHE", deadline_ms=lambda _: 2_000, + ) + self.assertEqual([item.value for item in result], list(range(100))) + self.assertTrue(all(item.ok and item.attempts == 1 for item in result)) + self.assertLessEqual(peak, 8) + self.assertEqual(sleeps, []) + self.assertEqual(executor.provider_policies["OKX"].requests_per_second, 5.0) + self.assertEqual(executor.provider_policies["BINANCE"].requests_per_second, 8.0) + async def test_identical_concurrent_work_is_singleflight(self): executor = BoundedWarmupExecutor[int, int]() started = asyncio.Event() @@ -1118,6 +1146,30 @@ async def test_partial_item_is_explicit_and_require_all_fails_closed(self): class SingleWarmupExecutionTests(unittest.IsolatedAsyncioTestCase): + async def test_query_service_uses_local_lane_only_on_backend_opt_in(self): + class Service(V2QueryService): + def __init__(self, local): + self.instruments = SimpleNamespace(get=lambda _: SimpleNamespace( + identity=SimpleNamespace(venue="OKX"))) + self.backend = SimpleNamespace(warmup_is_local=lambda _: local) + self.warmup_executor = BoundedWarmupExecutor() + self.last_batch_evidence = {} + + def warmup(self, requirement, *, purpose, request_id=None): + return "warmup-ok" + + requirement = DataRequirement( + instrument_uid=BINANCE_ETH, feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, source_policy_id="crypto_primary_v2", + interval="1m", warmup=WarmupSpecification.for_rows(1), + ) + for local, expected in ((True, "LOCAL_CANONICAL_CACHE"), (False, "OKX")): + service = Service(local) + self.assertEqual(await service.warmup_async( + requirement, purpose=AccessPurpose.INTERNAL_ALPHA), "warmup-ok") + self.assertEqual(set(service.warmup_executor._semaphores), {expected}) + self.assertEqual(set(service.warmup_executor._tokens), set() if local else {"OKX"}) + async def test_single_warmup_is_nonblocking_and_reuses_retry_policy(self): class Service(V2QueryService): def __init__(self): diff --git a/tests/test_phase113_reference_v2.py b/tests/test_phase113_reference_v2.py index b6f1634b..43565efb 100644 --- a/tests/test_phase113_reference_v2.py +++ b/tests/test_phase113_reference_v2.py @@ -132,6 +132,60 @@ async def fetch(self, request, *, capability, received_at_ns): self.active -= 1 +class StaleThenCurrentReferenceAdapter(FixtureReferenceAdapter): + """Return one delayed MARK/INDEX timestamp, then the current provider row.""" + + async def fetch(self, request, *, capability, received_at_ns): + fetched = await super().fetch( + request, + capability=capability, + received_at_ns=received_at_ns, + ) + if self.calls == 1: + self.observed_at_ns = NOW_NS + return fetched + + +class AdvanceClockAfterInitialBatchExecutor(BoundedWarmupExecutor): + """Model a snapshot aging only after all initial batch work completes.""" + + def __init__(self, *args, clock, adapter, **kwargs): + super().__init__(*args, **kwargs) + self._test_clock = clock + self._test_adapter = adapter + self._advance_once = True + + async def execute(self, *args, **kwargs): + result = await super().execute(*args, **kwargs) + if self._advance_once: + self._advance_once = False + self._test_clock["ns"] += 2_100_000_000 + self._test_adapter.observed_at_ns = self._test_clock["ns"] + return result + + +class AdvanceClockAfterBatchedMarkRefreshExecutor(BoundedWarmupExecutor): + """Expose stale response assembly after a multi-item MARK refresh. + + The first two-item execute ages initial snapshots. A legacy second + two-item refresh would age both refreshed snapshots before response + assembly. Per-item refreshes remain current at their own assembly turn. + """ + + def __init__(self, *args, clock, adapter, **kwargs): + super().__init__(*args, **kwargs) + self._test_clock = clock + self._test_adapter = adapter + + async def execute(self, items, **kwargs): + values = tuple(items) + result = await super().execute(values, **kwargs) + if len(values) == 2: + self._test_clock["ns"] += 2_100_000_000 + self._test_adapter.observed_at_ns = self._test_clock["ns"] + return result + + class RetryOnceReferenceAdapter(FixtureReferenceAdapter): """Deterministic admission pressure without bypassing ReferenceBatch.""" @@ -389,11 +443,12 @@ async def test_cached_reference_result_matches_by_value_not_python_identity(self async def test_stale_reference_result_fails_closed(self): registry = InstrumentRegistry() registry.register(self.binance, []) + adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - 2_000_000_000) stale = V2QueryService( instruments=InstrumentQuery(registry), backend=MemoryMarketDataBackend(), entitlements=grants(), - reference_batch=fixture_batch(observed_at_ns=NOW_NS - 2_000_000_000), + reference_batch=ReferenceBatch({("BINANCE", "USDM"): adapter}), reference_source_id=lambda item: f"{item.identity.venue}_DIRECT", clock_ns=lambda: NOW_NS, ) @@ -406,6 +461,333 @@ async def test_stale_reference_result_fails_closed(self): purpose=AccessPurpose.INTERNAL_ALPHA, ) self.assertEqual(result.results[0].problem.code.value, "DATA_STALE") + self.assertEqual(adapter.calls, 1) + + async def test_execution_mark_refreshes_a_cached_value_that_crossed_its_bound(self): + clock = {"ns": NOW_NS} + adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - 1_500_000_000) + registry = InstrumentRegistry() + registry.register(self.binance, []) + execution_grants = EntitlementPolicy(( + EntitlementGrant( + source_id="BINANCE_DIRECT", + license_revision="phase113-test", + purposes=frozenset({AccessPurpose.INTERNAL_EXECUTION}), + products=frozenset({DataProduct.CANONICAL_SNAPSHOT}), + valid_from_ns=0, + ), + )) + service = V2QueryService( + instruments=InstrumentQuery(registry), + backend=MemoryMarketDataBackend(), + entitlements=execution_grants, + reference_batch=ReferenceBatch( + {("BINANCE", "USDM"): adapter}, + clock_ns=lambda: clock["ns"], + # Keep the TTL valid so this proves freshness bypass rather + # than ordinary cache expiry. + monotonic=lambda: 0.0, + ), + reference_source_id=lambda _item: "BINANCE_DIRECT", + clock_ns=lambda: clock["ns"], + ) + requirement = ReferenceDataRequirement( + instrument_uid=self.binance.instrument_uid, + product=ReferenceProduct.MARK_INDEX_PRICE, + consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id="crypto_liquid_v2", + limit=1, + page_size=1, + max_pages=1, + max_freshness_ms=2_000, + ) + first = await service.reference_data_batch_async( + ReferenceBatchRequirement("phase113-execution", (requirement,)), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertFalse(first.partial) + self.assertEqual(adapter.calls, 1) + + # The cached source has now crossed 2s, but the refreshed provider + # observation is current. A cache hit must not turn it into stale data. + clock["ns"] += 600_000_000 + adapter.observed_at_ns = clock["ns"] + second = await service.reference_data_batch_async( + ReferenceBatchRequirement("phase113-execution", (requirement,)), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertFalse(second.partial) + self.assertEqual(adapter.calls, 2) + self.assertFalse(second.results[0].result.cache_hit) + + async def test_execution_mark_refreshes_an_aged_batch_receipt_once(self): + clock = {"ns": NOW_NS} + adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS) + registry = InstrumentRegistry() + registry.register(self.binance, []) + execution_grants = EntitlementPolicy(( + EntitlementGrant( + source_id="BINANCE_DIRECT", + license_revision="phase113-test", + purposes=frozenset({AccessPurpose.INTERNAL_EXECUTION}), + products=frozenset({DataProduct.CANONICAL_SNAPSHOT}), + valid_from_ns=0, + ), + )) + executor = AdvanceClockAfterInitialBatchExecutor( + clock=clock, + adapter=adapter, + ) + service = V2QueryService( + instruments=InstrumentQuery(registry), + backend=MemoryMarketDataBackend(), + entitlements=execution_grants, + warmup_executor=executor, + reference_batch=ReferenceBatch( + {("BINANCE", "USDM"): adapter}, + clock_ns=lambda: clock["ns"], + monotonic=lambda: 0.0, + ), + reference_source_id=lambda _item: "BINANCE_DIRECT", + clock_ns=lambda: clock["ns"], + ) + requirement = ReferenceDataRequirement( + instrument_uid=self.binance.instrument_uid, + product=ReferenceProduct.MARK_INDEX_PRICE, + consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id="crypto_liquid_v2", + limit=1, + page_size=1, + max_pages=1, + max_freshness_ms=2_000, + ) + result = await service.reference_data_batch_async( + ReferenceBatchRequirement("phase113-execution", (requirement,)), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertFalse(result.partial) + self.assertEqual(adapter.calls, 2) + + async def test_execution_marks_refresh_and_validate_each_item_at_assembly(self): + clock = {"ns": NOW_NS} + adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS) + second = record(venue="BINANCE", market="USDM", symbol="ETHUSDT", base="ETH") + registry = InstrumentRegistry() + registry.register(self.binance, []) + registry.register(second, []) + execution_grants = EntitlementPolicy(( + EntitlementGrant( + source_id="BINANCE_DIRECT", + license_revision="phase113-test", + purposes=frozenset({AccessPurpose.INTERNAL_EXECUTION}), + products=frozenset({DataProduct.CANONICAL_SNAPSHOT}), + valid_from_ns=0, + ), + )) + executor = AdvanceClockAfterBatchedMarkRefreshExecutor( + clock=clock, + adapter=adapter, + ) + service = V2QueryService( + instruments=InstrumentQuery(registry), + backend=MemoryMarketDataBackend(), + entitlements=execution_grants, + warmup_executor=executor, + reference_batch=ReferenceBatch( + {("BINANCE", "USDM"): adapter}, + clock_ns=lambda: clock["ns"], + monotonic=lambda: 0.0, + ), + reference_source_id=lambda _item: "BINANCE_DIRECT", + clock_ns=lambda: clock["ns"], + ) + + def mark_requirement(instrument_uid: str) -> ReferenceDataRequirement: + return ReferenceDataRequirement( + instrument_uid=instrument_uid, + product=ReferenceProduct.MARK_INDEX_PRICE, + consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id="crypto_liquid_v2", + limit=1, + page_size=1, + max_pages=1, + max_freshness_ms=2_000, + ) + + result = await service.reference_data_batch_async( + ReferenceBatchRequirement( + "phase113-execution", + ( + mark_requirement(self.binance.instrument_uid), + mark_requirement(second.instrument_uid), + ), + require_all=False, + ), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + + self.assertFalse(result.partial) + self.assertEqual(adapter.calls, 4) + self.assertTrue(all(item.problem is None for item in result.results)) + self.assertEqual( + [item.result.request.instrument.instrument_uid for item in result.results], + [self.binance.instrument_uid, second.instrument_uid], + ) + + async def test_shared_execution_mark_refresh_uses_one_provider_call(self): + clock = {"ns": NOW_NS} + adapter = FixtureReferenceAdapter( + delay_seconds=0.02, + observed_at_ns=NOW_NS - 1_500_000_000, + ) + registry = InstrumentRegistry() + registry.register(self.binance, []) + execution_grants = EntitlementPolicy(( + EntitlementGrant( + source_id="BINANCE_DIRECT", + license_revision="phase113-test", + purposes=frozenset({AccessPurpose.INTERNAL_EXECUTION}), + products=frozenset({DataProduct.CANONICAL_SNAPSHOT}), + valid_from_ns=0, + ), + )) + service = V2QueryService( + instruments=InstrumentQuery(registry), + backend=MemoryMarketDataBackend(), + entitlements=execution_grants, + reference_batch=ReferenceBatch( + {("BINANCE", "USDM"): adapter}, + clock_ns=lambda: clock["ns"], + monotonic=lambda: 0.0, + ), + reference_source_id=lambda _item: "BINANCE_DIRECT", + clock_ns=lambda: clock["ns"], + ) + requirement = ReferenceDataRequirement( + instrument_uid=self.binance.instrument_uid, + product=ReferenceProduct.MARK_INDEX_PRICE, + consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id="crypto_liquid_v2", + limit=1, + page_size=1, + max_pages=1, + max_freshness_ms=2_000, + ) + initial = await service.reference_data_batch_async( + ReferenceBatchRequirement("phase113-execution", (requirement,)), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertFalse(initial.partial) + clock["ns"] += 600_000_000 + adapter.observed_at_ns = clock["ns"] + first, second = await asyncio.gather( + service.reference_data_batch_async( + ReferenceBatchRequirement("phase113-execution", (requirement,)), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ), + service.reference_data_batch_async( + ReferenceBatchRequirement("phase113-execution", (requirement,)), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ), + ) + self.assertFalse(first.partial) + self.assertFalse(second.partial) + self.assertEqual(adapter.calls, 2) + + async def test_execution_mark_refreshes_a_transient_stale_provider_value_once(self): + clock = {"ns": NOW_NS} + adapter = StaleThenCurrentReferenceAdapter( + observed_at_ns=NOW_NS - 2_100_000_000 + ) + registry = InstrumentRegistry() + registry.register(self.binance, []) + execution_grants = EntitlementPolicy(( + EntitlementGrant( + source_id="BINANCE_DIRECT", + license_revision="phase113-test", + purposes=frozenset({AccessPurpose.INTERNAL_EXECUTION}), + products=frozenset({DataProduct.CANONICAL_SNAPSHOT}), + valid_from_ns=0, + ), + )) + service = V2QueryService( + instruments=InstrumentQuery(registry), + backend=MemoryMarketDataBackend(), + entitlements=execution_grants, + reference_batch=ReferenceBatch( + {("BINANCE", "USDM"): adapter}, + clock_ns=lambda: clock["ns"], + ), + reference_source_id=lambda _item: "BINANCE_DIRECT", + clock_ns=lambda: clock["ns"], + ) + requirement = ReferenceDataRequirement( + instrument_uid=self.binance.instrument_uid, + product=ReferenceProduct.MARK_INDEX_PRICE, + consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id="crypto_liquid_v2", + limit=1, + page_size=1, + max_pages=1, + max_freshness_ms=2_000, + ) + result = await service.reference_data_batch_async( + ReferenceBatchRequirement( + "phase113-execution", + (requirement,), + require_all=False, + ), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertFalse(result.partial) + self.assertEqual(adapter.calls, 2) + self.assertEqual(result.results[0].result.observations[0].observed_at_ns, NOW_NS) + + async def test_execution_mark_keeps_stale_provider_value_fail_closed_after_one_refresh(self): + clock = {"ns": NOW_NS} + adapter = FixtureReferenceAdapter(observed_at_ns=NOW_NS - 2_100_000_000) + registry = InstrumentRegistry() + registry.register(self.binance, []) + execution_grants = EntitlementPolicy(( + EntitlementGrant( + source_id="BINANCE_DIRECT", + license_revision="phase113-test", + purposes=frozenset({AccessPurpose.INTERNAL_EXECUTION}), + products=frozenset({DataProduct.CANONICAL_SNAPSHOT}), + valid_from_ns=0, + ), + )) + service = V2QueryService( + instruments=InstrumentQuery(registry), + backend=MemoryMarketDataBackend(), + entitlements=execution_grants, + reference_batch=ReferenceBatch( + {("BINANCE", "USDM"): adapter}, + clock_ns=lambda: clock["ns"], + ), + reference_source_id=lambda _item: "BINANCE_DIRECT", + clock_ns=lambda: clock["ns"], + ) + requirement = ReferenceDataRequirement( + instrument_uid=self.binance.instrument_uid, + product=ReferenceProduct.MARK_INDEX_PRICE, + consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id="crypto_liquid_v2", + limit=1, + page_size=1, + max_pages=1, + max_freshness_ms=2_000, + ) + result = await service.reference_data_batch_async( + ReferenceBatchRequirement( + "phase113-execution", + (requirement,), + require_all=False, + ), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertEqual(result.results[0].problem.code.value, "DATA_STALE") + self.assertEqual(adapter.calls, 2) async def test_retryable_reference_result_reuses_shared_warmup_policy(self): adapter = RetryOnceReferenceAdapter() diff --git a/tests/test_phase115_universal_release.py b/tests/test_phase115_universal_release.py index 2538750c..20268bc0 100644 --- a/tests/test_phase115_universal_release.py +++ b/tests/test_phase115_universal_release.py @@ -336,6 +336,29 @@ def test_consumer_route_binding_is_deterministic_isolated_and_round_trips(self): manifest.digest, ) + def test_generation_bound_consumer_route_binding_round_trips_and_keeps_v1_compatibility(self): + manifest, *_ = _manifest() + consumer_id = manifest.products[0].consumer_id + generation_bound = ConsumerRouteBinding.from_manifest( + manifest, + consumer_id=consumer_id, + consumer_manifest_revision=7, + ) + self.assertEqual( + generation_bound.canonical_mapping()["schema"], + "qdl.v2.consumer-route-binding.v2", + ) + self.assertEqual(generation_bound.consumer_manifest_revision, 7) + self.assertEqual( + ConsumerRouteBinding.from_canonical_mapping( + generation_bound.canonical_mapping() + ).consumer_manifest_revision, + 7, + ) + legacy = ConsumerRouteBinding.from_manifest(manifest, consumer_id=consumer_id) + self.assertEqual(legacy.canonical_mapping()["schema"], "qdl.v2.consumer-route-binding.v1") + self.assertIsNone(legacy.consumer_manifest_revision) + def test_consumer_route_binding_rejects_unknown_tampered_and_duplicate_routes(self): manifest, *_ = _manifest() with self.assertRaisesRegex(ValueError, "no admitted product"): @@ -419,6 +442,33 @@ def test_pure_renderer_writes_only_one_named_consumer_binding(self): self.assertEqual(parsed.consumer_id, consumer_id) self.assertTrue(all(item.consumer_id == consumer_id for item in parsed.products)) + def test_renderer_seals_consumer_manifest_revision_when_given_manifest(self): + manifest, *_ = _manifest() + consumer_id = manifest.products[0].consumer_id + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + artifact = root / "release.json" + consumer_manifest = root / "consumer.yaml" + output = root / "consumer-binding.json" + artifact.write_text(json.dumps({ + "schema": "qdl.phase115.universal-release-preflight.v1", + "release_manifest": manifest.canonical_mapping(), + "release_summary": manifest.report_payload(), + }), encoding="utf-8") + consumer_manifest.write_text(yaml.safe_dump({ + "metadata": {"id": consumer_id, "revision": 7}, + }), encoding="utf-8") + self.assertEqual(render_consumer_binding([ + "--release-artifact", str(artifact), + "--consumer-id", consumer_id, + "--consumer-manifest", str(consumer_manifest), + "--output", str(output), + ]), 0) + parsed = ConsumerRouteBinding.from_canonical_mapping( + json.loads(output.read_text(encoding="utf-8")) + ) + self.assertEqual(parsed.consumer_manifest_revision, 7) + class UniversalReleaseEvidenceTests(unittest.TestCase): def test_evidence_bundle_requires_one_exact_admission_generation(self): diff --git a/tests/test_phase115c_bar_edge_schedule.py b/tests/test_phase115c_bar_edge_schedule.py index fe353677..c3cfa0ae 100644 --- a/tests/test_phase115c_bar_edge_schedule.py +++ b/tests/test_phase115c_bar_edge_schedule.py @@ -10,6 +10,7 @@ def _edge(*sources: SimpleNamespace) -> StableBinanceBarEdge: edge = object.__new__(StableBinanceBarEdge) + edge.repair_only = False edge._last_open_ms = {} edge.bindings = tuple((source, SimpleNamespace(runtime="BINANCE")) for source in sources) edge.okx_bindings = () @@ -35,6 +36,7 @@ def wait(self, seconds: float) -> bool: return True edge = object.__new__(StableBinanceBarEdge) + edge.repair_only = False edge._history_bootstrap_active = True edge._history_bootstrapped = False edge._rest_fallback_active = False diff --git a/tests/test_phase115c_five_liquid_handoff.py b/tests/test_phase115c_five_liquid_handoff.py index 58c6c368..1b81587f 100644 --- a/tests/test_phase115c_five_liquid_handoff.py +++ b/tests/test_phase115c_five_liquid_handoff.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections import Counter +from dataclasses import replace from pathlib import Path import tempfile from types import SimpleNamespace @@ -21,7 +22,7 @@ ) from qdl.runtime.production_catalog import ProductionDemandManifest from qdl_sdk import DataRequirement, Feed, FeedStatusResponse, Grade, StalePolicy -from scripts.phase103_consumer_receipt_acceptance import _quiet_trade_status_is_observable +from scripts.phase103_consumer_receipt_acceptance import _quiet_continuity_status_is_observable ROOT = Path(__file__).resolve().parents[1] @@ -249,7 +250,7 @@ def _trade_requirement(instrument_uid: str) -> DataRequirement: def _product(requirement: DataRequirement) -> SimpleNamespace: return SimpleNamespace( delivery=DeliveryClass.DURABLE, - feed=Feed.TRADE, + feed=requirement.feed, requirement=requirement, ) @@ -264,7 +265,7 @@ def _status( return FeedStatusResponse.model_validate({ "schema": "qdl.feed-status.v2", "instrument_uid": instrument_uid or requirement.instrument_uid, - "feed": "TRADE", + "feed": requirement.feed.value, "quality": { "state": "LIVE", "freshness_ms": 15_001, @@ -286,7 +287,7 @@ def test_five_okx_trade_statuses_are_quiet_safe_and_symbol_isolated(self) -> Non requirement = self._trade_requirement(instrument_uid) product = self._product(requirement) self.assertTrue( - _quiet_trade_status_is_observable( + _quiet_continuity_status_is_observable( product, requirement, self._status(requirement) ) ) @@ -294,7 +295,7 @@ def test_five_okx_trade_statuses_are_quiet_safe_and_symbol_isolated(self) -> Non value for value in OKX_TRADE_UIDS.values() if value != instrument_uid ) self.assertFalse( - _quiet_trade_status_is_observable( + _quiet_continuity_status_is_observable( product, requirement, self._status(requirement, instrument_uid=other_uid), @@ -306,7 +307,7 @@ def test_five_okx_trade_statuses_fail_closed_for_disconnect_or_gap(self) -> None with self.subTest(symbol=symbol, failure="disconnected"): requirement = self._trade_requirement(instrument_uid) self.assertFalse( - _quiet_trade_status_is_observable( + _quiet_continuity_status_is_observable( self._product(requirement), requirement, self._status( @@ -317,13 +318,59 @@ def test_five_okx_trade_statuses_fail_closed_for_disconnect_or_gap(self) -> None with self.subTest(symbol=symbol, failure="gap"): requirement = self._trade_requirement(instrument_uid) self.assertFalse( - _quiet_trade_status_is_observable( + _quiet_continuity_status_is_observable( self._product(requirement), requirement, self._status(requirement, gap_open=True), ) ) + def test_five_okx_book_delta_statuses_are_quiet_safe_and_fail_closed(self) -> None: + for symbol, instrument_uid in OKX_TRADE_UIDS.items(): + with self.subTest(symbol=symbol, result="quiet-live"): + requirement = replace( + self._trade_requirement(instrument_uid), + feed=Feed.BOOK_DELTA, + max_freshness_ms=2_000, + ) + product = self._product(requirement) + self.assertTrue( + _quiet_continuity_status_is_observable( + product, requirement, self._status(requirement) + ) + ) + blocked = replace(requirement, event_recency_policy=None) + self.assertFalse( + _quiet_continuity_status_is_observable( + self._product(blocked), blocked, self._status(blocked) + ) + ) + for provider_session_state, gap_open in ( + ("DISCONNECTED", False), + ("LIVE", True), + ): + with self.subTest( + symbol=symbol, + provider_session_state=provider_session_state, + gap_open=gap_open, + ): + requirement = replace( + self._trade_requirement(instrument_uid), + feed=Feed.BOOK_DELTA, + max_freshness_ms=2_000, + ) + self.assertFalse( + _quiet_continuity_status_is_observable( + self._product(requirement), + requirement, + self._status( + requirement, + provider_session_state=provider_session_state, + gap_open=gap_open, + ), + ) + ) + def test_five_okx_trade_reconnect_rejects_duplicate_or_stale_cursor(self) -> None: """Each symbol keeps a strictly advancing durable resume cursor.""" for index, symbol in enumerate(OKX_TRADE_UIDS): diff --git a/tests/test_phase12_bound_bar_edge_projection.py b/tests/test_phase12_bound_bar_edge_projection.py index ca853e0c..4a4c9cf7 100644 --- a/tests/test_phase12_bound_bar_edge_projection.py +++ b/tests/test_phase12_bound_bar_edge_projection.py @@ -1,6 +1,7 @@ from __future__ import annotations from copy import deepcopy +from dataclasses import replace import importlib.util from pathlib import Path import sys @@ -85,6 +86,54 @@ def _binding_for_source_ids(*binding_ids: str) -> ConsumerRouteBinding: ) +def _alpha_binding_for_source_ids( + consumer_id: str, *binding_ids: str +) -> ConsumerRouteBinding: + base = _binding_for_source_ids(*binding_ids) + products = tuple( + replace( + item, + consumer_id=consumer_id, + consumer_class=UniversalConsumerClass.SINGLE_SYMBOL_ALPHA, + require_live=False, + execution_grade=False, + ) + for item in base.products + ) + return replace( + base, + consumer_id=consumer_id, + consumer_class=UniversalConsumerClass.SINGLE_SYMBOL_ALPHA, + products=products, + binding_sha256=None, + ) + + +def _retained_projection(*binding_ids: str) -> dict: + return { + "schema": "qdl.phase12.bound-bar-edge-projection.v1", + "status": "MATERIALIZED", + "consumer_id": "trading-system.paper.stable", + "binding_ids": list(binding_ids), + "catalog_sha256": "c" * 64, + "acquisition_sha256": "d" * 64, + } + + +def _retained_union_projection(*binding_ids: str) -> dict: + return { + "schema": "qdl.phase12.bound-bar-edge-projection.v1", + "status": "MATERIALIZED", + "consumer_ids": [ + "alpha.binance.paper.stable", + "trading-system.paper.stable", + ], + "binding_ids": list(binding_ids), + "catalog_sha256": "c" * 64, + "acquisition_sha256": "d" * 64, + } + + class BoundBarEdgeProjectionTests(unittest.TestCase): def test_projects_exact_final_bar_routes_without_mutating_inputs(self) -> None: catalog, acquisition = _documents() @@ -130,6 +179,128 @@ def test_projects_exact_final_bar_routes_without_mutating_inputs(self) -> None: selected["okx-swap-eth-usdt-swap-bar-1m"]["business_websocket_url"] ) + def test_unions_alpha_bindings_and_keeps_final_bars_without_live_tick_requirement(self) -> None: + catalog, acquisition = _documents() + first = _alpha_binding_for_source_ids( + "alpha.first.binance.paper", + "binance-usdm-ethusdt-bar-15m", + ) + duplicate = _alpha_binding_for_source_ids( + "alpha.second.binance.paper", + "binance-usdm-ethusdt-bar-15m", + ) + okx = _alpha_binding_for_source_ids( + "alpha.first.okx.no-order", + "okx-swap-eth-usdt-swap-bar-15m", + ) + + result = projection.build_bound_bar_projection_set( + bindings=(first, duplicate, okx), + catalog_document=catalog, + acquisition_document=acquisition, + ) + + self.assertEqual(result.summary["bar_route_count"], 2) + self.assertEqual( + result.summary["binding_ids"], + [ + "binance-usdm-ethusdt-bar-15m", + "okx-swap-eth-usdt-swap-bar-15m", + ], + ) + self.assertEqual( + result.summary["consumer_ids"], + [ + "alpha.first.binance.paper", + "alpha.first.okx.no-order", + "alpha.second.binance.paper", + ], + ) + self.assertNotIn("consumer_id", result.summary) + + def test_retains_active_baseline_when_materializing_alpha_union(self) -> None: + catalog, acquisition = _documents() + alpha = _alpha_binding_for_source_ids( + "alpha.first.binance.paper", + "binance-usdm-ethusdt-bar-15m", + ) + + result = projection.build_bound_bar_projection_set( + bindings=(alpha,), + catalog_document=catalog, + acquisition_document=acquisition, + retained_projection=_retained_projection("binance-usdm-btcusdt-bar-1m"), + retained_projection_sha256="e" * 64, + ) + + self.assertEqual( + result.summary["binding_ids"], + ["binance-usdm-btcusdt-bar-1m", "binance-usdm-ethusdt-bar-15m"], + ) + self.assertEqual( + result.summary["retained_projection"], + { + "consumer_id": "trading-system.paper.stable", + "binding_ids": ["binance-usdm-btcusdt-bar-1m"], + "sha256": "e" * 64, + }, + ) + + def test_rejects_retained_route_that_is_missing(self) -> None: + catalog, acquisition = _documents() + alpha = _alpha_binding_for_source_ids( + "alpha.first.binance.paper", + "binance-usdm-ethusdt-bar-15m", + ) + + with self.assertRaisesRegex(ValueError, "no longer resolves"): + projection.build_bound_bar_projection_set( + bindings=(alpha,), + catalog_document=catalog, + acquisition_document=acquisition, + retained_projection=_retained_projection("missing-bar-route"), + ) + + def test_rematerializes_multi_consumer_projection_with_okx_rest_finality(self) -> None: + catalog, acquisition = _documents() + alpha = _alpha_binding_for_source_ids( + "alpha.first.binance.paper", + "binance-usdm-ethusdt-bar-15m", + ) + + result = projection.build_bound_bar_projection_set( + bindings=(alpha,), + catalog_document=catalog, + acquisition_document=acquisition, + retained_projection=_retained_union_projection( + "binance-usdm-btcusdt-bar-1m", + "okx-swap-eth-usdt-swap-bar-1m", + ), + retained_projection_sha256="e" * 64, + ) + + selected = { + item["binding_id"]: item for item in result.acquisition["bindings"] + } + self.assertEqual( + selected["okx-swap-eth-usdt-swap-bar-1m"]["mode"], + "PYTHON_REST", + ) + self.assertEqual( + result.summary["retained_projection"], + { + "consumer_ids": [ + "alpha.binance.paper.stable", + "trading-system.paper.stable", + ], + "binding_ids": [ + "binance-usdm-btcusdt-bar-1m", + "okx-swap-eth-usdt-swap-bar-1m", + ], + "sha256": "e" * 64, + }, + ) + def test_rejects_native_acquisition_outside_declared_okx_final_bar_recovery(self) -> None: catalog, acquisition = _documents() binding = _binding_for_source_ids("binance-usdm-btcusdt-bar-1m") diff --git a/tests/test_phase533_alpha_runtime_entitlements.py b/tests/test_phase533_alpha_runtime_entitlements.py new file mode 100644 index 00000000..fa52a139 --- /dev/null +++ b/tests/test_phase533_alpha_runtime_entitlements.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +from copy import deepcopy +import importlib.util +from pathlib import Path +import unittest + +import yaml + +from qdl.consumer import ConsumerManifestLoader, requirement_key +from qdl.query import ConsumerGrade, FeedType + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = ROOT / "scripts/phase533_materialize_alpha_runtime_entitlements.py" + + +def _module(): + spec = importlib.util.spec_from_file_location("phase533_alpha_entitlements", SCRIPT_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _load(path: Path): + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +class Phase533AlphaRuntimeEntitlementTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.tool = _module() + cls.catalog = _load(ROOT / "config/v2/stable-source-bindings.yaml") + cls.demand = _load(ROOT / "config/v2/stable-crypto-demand.yaml") + cls.reference = _load(ROOT / "consumers/stable/reference-l2-stable.yaml") + cls.manifests = { + consumer_id: _load(ROOT / "consumers/stable" / target["filename"]) + for consumer_id, target in cls.tool._TARGETS.items() + } + cls.release = _load(ROOT / "config/v2/stable-v2-release-routing.yaml") + cls.primary = _load(ROOT / "config/v2/stable-primary-consumer-routing.yaml") + ( + cls.rendered, + cls.rendered_release, + cls.rendered_primary, + cls.summary, + ) = cls.tool.build_documents( + catalog=cls.catalog, + demand=cls.demand, + reference_manifest=cls.reference, + alpha_manifests=cls.manifests, + release_route=cls.release, + primary_route=cls.primary, + ) + + def test_five_liquid_manifests_are_complete_bounded_and_non_execution(self) -> None: + expected = { + "alpha.binance.paper.stable": { + FeedType.BAR: 70, + FeedType.TRADE: 5, + FeedType.QUOTE: 5, + FeedType.BOOK_SNAPSHOT: 5, + FeedType.BOOK_DELTA: 5, + "reference": 35, + "total": 125, + }, + "alpha.okx.paper.stable": { + FeedType.BAR: 70, + FeedType.TRADE: 5, + FeedType.QUOTE: 5, + FeedType.BOOK_SNAPSHOT: 5, + FeedType.BOOK_DELTA: 5, + "reference": 20, + "total": 110, + }, + } + reference_feeds = { + FeedType.BASIS, + FeedType.CONTRACT_METADATA, + FeedType.FUNDING_RATE, + FeedType.LONG_SHORT_RATIO, + FeedType.MARK_INDEX_PRICE, + FeedType.OPEN_INTEREST, + FeedType.TAKER_FLOW, + } + for consumer_id, payload in self.rendered.items(): + manifest = ConsumerManifestLoader.from_mapping(payload) + wanted = expected[consumer_id] + self.assertEqual(len(manifest.requirements), wanted["total"]) + self.assertEqual(manifest.execution_dependency, "FORBIDDEN") + self.assertEqual(manifest.quotas.max_warmup_rows, 10_000) + self.assertEqual({item.consumer_grade for item in manifest.requirements}, {ConsumerGrade.ALPHA}) + for feed in ( + FeedType.BAR, + FeedType.TRADE, + FeedType.QUOTE, + FeedType.BOOK_SNAPSHOT, + FeedType.BOOK_DELTA, + ): + self.assertEqual(sum(item.feed is feed for item in manifest.requirements), wanted[feed]) + references = [item for item in manifest.requirements if item.feed in reference_feeds] + self.assertEqual(len(references), wanted["reference"]) + self.assertTrue(all(item.source_policy_id == "crypto_liquid_v2" for item in references)) + self.assertTrue(all(item.recovery.value == "SNAPSHOT_AND_REPLAY" for item in manifest.requirements if item.feed is FeedType.BAR)) + self.assertTrue(all(item.require_final_bars for item in manifest.requirements if item.feed is FeedType.BAR)) + self.assertTrue(all(item.max_session_liveness_ms == 45_000 for item in manifest.requirements if item.feed in {FeedType.TRADE, FeedType.QUOTE, FeedType.BOOK_SNAPSHOT, FeedType.BOOK_DELTA})) + + def test_native_identity_interval_and_venue_never_cross_mix(self) -> None: + catalog_by_uid = { + item["instrument_uid"]: item for item in self.catalog["instruments"] + } + expected_venue = { + "alpha.binance.paper.stable": ("BINANCE", "USDM"), + "alpha.okx.paper.stable": ("OKX", "SWAP"), + } + for consumer_id, payload in self.rendered.items(): + manifest = ConsumerManifestLoader.from_mapping(payload) + venue, market = expected_venue[consumer_id] + bars_by_uid: dict[str, set[str]] = {} + for requirement in manifest.requirements: + instrument = catalog_by_uid[requirement.instrument_uid] + self.assertEqual((instrument["venue"], instrument["market"]), (venue, market)) + if requirement.feed is FeedType.BAR: + bars_by_uid.setdefault(requirement.instrument_uid, set()).add(requirement.interval) + self.assertEqual(len(bars_by_uid), 5) + self.assertTrue(all(len(intervals) == 14 for intervals in bars_by_uid.values())) + + def test_release_routes_are_complete_and_only_trade_can_fallback(self) -> None: + routes = { + item["consumer_id"]: item + for item in self.rendered_release["consumers"] + if item["consumer_id"] in self.rendered + } + for consumer_id, payload in self.rendered.items(): + manifest = ConsumerManifestLoader.from_mapping(payload) + products = { + item["requirement_key"]: item for item in routes[consumer_id]["products"] + } + self.assertEqual(set(products), {requirement_key(item) for item in manifest.requirements}) + for requirement in manifest.requirements: + product = products[requirement_key(requirement)] + self.assertEqual(product["route"], "V2_PRIMARY") + if requirement.feed is FeedType.TRADE and consumer_id.startswith("alpha.binance"): + self.assertEqual(product["fallback"], "V1") + else: + self.assertEqual(product["fallback"], "BLOCKED") + + def test_render_is_idempotent_and_capacity_remains_bounded(self) -> None: + rerun = self.tool.build_documents( + catalog=self.catalog, + demand=self.demand, + reference_manifest=self.reference, + alpha_manifests=self.rendered, + release_route=self.rendered_release, + primary_route=self.rendered_primary, + ) + self.assertEqual(rerun[0], self.rendered) + self.assertEqual(rerun[1], self.rendered_release) + self.assertEqual(rerun[2], self.rendered_primary) + self.assertFalse(rerun[3]["manifest_changed"]) + self.assertFalse(rerun[3]["release_route_changed"]) + + oversized = deepcopy(self.rendered["alpha.binance.paper.stable"]) + oversized["spec"]["requirements"] = ( + oversized["spec"]["requirements"] * 3 + )[:257] + with self.assertRaisesRegex(ValueError, "1..256"): + ConsumerManifestLoader.from_mapping(oversized) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_phase533_query_readiness.py b/tests/test_phase533_query_readiness.py new file mode 100644 index 00000000..ac5a17d4 --- /dev/null +++ b/tests/test_phase533_query_readiness.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import asyncio +import inspect +import sqlite3 +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from qdl.runtime.readiness import ComponentState +from qdl.runtime.stable import ( + build_stable_spool, + serve_stable_projector, + stable_spool_capacity, + stable_readiness, +) +from qdl.transport import DurableEvent, SQLiteDurableSpool, SpoolConfig + + +def _event(index: int, payload: bytes) -> DurableEvent: + return DurableEvent( + stream="md.canonical.v2.trade", + partition_key="BINANCE.USDM.PERPETUAL.BTC-USDT/trade", + event_id=index.to_bytes(16, "big"), + payload=payload, + accepted_at_ns=index, + content_type="application/json", + ) + + +class _Quota: + def ping(self) -> bool: + return True + + +class _Registry: + count = 1 + revisions = (("alpha.binance.paper.stable", 8),) + + +class Phase533QueryReadinessTests(unittest.TestCase): + def setUp(self) -> None: + self.temp = tempfile.TemporaryDirectory(prefix="qdl-phase533-readiness-") + self.spool_config = SpoolConfig( + path=Path(self.temp.name) / "cache.sqlite3", + min_free_disk_bytes=0, + max_partition_records=1, + ) + self.spool = SQLiteDurableSpool(self.spool_config) + self.runtime_config = SimpleNamespace( + role="query_v2", + authority_mode="RUST_PRIMARY", + authority_revision=10, + config_revision="phase533-test", + ) + + def tearDown(self) -> None: + self.spool.close() + self.temp.cleanup() + + def test_readiness_summary_tracks_transactional_usage_after_trim(self): + self.spool.append(_event(1, b"first")) + self.spool.append(_event(2, b"second")) + + summary = self.spool.readiness_summary() + + self.assertEqual(summary.records, 1) + self.assertEqual(summary.payload_bytes, len(b"second")) + + def test_existing_usage_state_reopens_without_event_aggregate(self): + self.spool.append(_event(1, b"first")) + self.spool.close() + + with patch.object( + SQLiteDurableSpool, + "_aggregate_event_usage_locked", + side_effect=AssertionError("live cache must not be rescanned"), + ): + reopened = SQLiteDurableSpool(self.spool_config) + try: + self.assertEqual(reopened.readiness_summary().records, 1) + finally: + reopened.close() + + def test_missing_usage_state_is_reconstructed_once(self): + self.spool.append(_event(1, b"first")) + self.spool.append(_event(2, b"second")) + self.spool.close() + with sqlite3.connect(self.spool_config.path) as connection: + connection.execute("DELETE FROM spool_state") + + reopened = SQLiteDurableSpool(self.spool_config) + try: + summary = reopened.readiness_summary() + self.assertEqual(summary.records, 1) + self.assertEqual(summary.payload_bytes, len(b"second")) + finally: + reopened.close() + + def test_default_spool_open_retains_integrity_check(self): + path = Path(self.temp.name) / "default-integrity.sqlite3" + with patch.object( + SQLiteDurableSpool, "integrity_check", return_value=True + ) as check: + spool = SQLiteDurableSpool(SpoolConfig(path=path, min_free_disk_bytes=0)) + try: + check.assert_called_once_with() + finally: + spool.close() + + def test_catalog_derived_spool_capacity_covers_all_physical_windows(self): + catalog = SimpleNamespace(bindings=tuple( + SimpleNamespace(partition_key=f"physical/{index}") for index in range(101) + )) + + capacity = stable_spool_capacity(catalog) + + self.assertEqual(capacity.physical_partitions, 101) + # The public query/SDK ceiling stays 10,000. The extra bounded + # physical tail prevents late backfills from evicting current rows by + # logical append order before market-time selection. + self.assertEqual(capacity.max_partition_records, 10_064) + self.assertEqual(capacity.max_records, 1_016_464) + + def test_rebuildable_stable_spool_skips_open_time_integrity_scan(self): + stable_config = SimpleNamespace( + durable_state_dir=Path(self.temp.name) / "stable", + cursor_ttl_seconds=60, + ) + catalog = SimpleNamespace(bindings=(SimpleNamespace(partition_key="physical/one"),)) + with patch.object( + SQLiteDurableSpool, + "_validate_integrity", + side_effect=AssertionError("stable startup must not full-scan cache"), + ): + spool = build_stable_spool(stable_config, catalog) + try: + self.assertFalse(spool.config.verify_integrity_on_open) + self.assertEqual(spool.config.max_records, 1_000_000) + self.assertTrue(spool.integrity_check()) + finally: + spool.close() + + def test_open_does_not_rebuild_retention_index(self): + path = Path(self.temp.name) / "no-reindex.sqlite3" + initial = SQLiteDurableSpool(SpoolConfig(path=path, min_free_disk_bytes=0)) + initial.close() + + statements: list[str] = [] + original_connect = sqlite3.connect + + class TracedConnection(sqlite3.Connection): + def executescript(self, script): + statements.append(script) + return super().executescript(script) + + with patch( + "qdl.transport.sqlite_spool.sqlite3.connect", + side_effect=lambda *args, **kwargs: original_connect( + *args, factory=TracedConnection, **kwargs + ), + ): + reopened = SQLiteDurableSpool( + SpoolConfig(path=path, min_free_disk_bytes=0) + ) + try: + self.assertTrue( + any("CREATE INDEX IF NOT EXISTS idx_qdl_spool_events_retention" in sql for sql in statements) + ) + self.assertFalse(any("DROP INDEX" in sql for sql in statements)) + finally: + reopened.close() + + def test_projector_bootstrap_uses_bounded_usage_summary(self): + source = inspect.getsource(serve_stable_projector) + self.assertIn("spool.readiness_summary", source) + self.assertNotIn("spool.stats", source) + + def test_stable_readiness_uses_bounded_summary_not_full_stats(self): + self.spool.append(_event(1, b"payload")) + readiness = stable_readiness( + self.runtime_config, _Registry(), self.spool, quota=_Quota() + ) + + with patch.object(self.spool, "stats", side_effect=AssertionError("full scan")): + snapshot = asyncio.run(readiness.snapshot()) + + component = next(item for item in snapshot.components if item.name == "query_cache") + self.assertTrue(snapshot.ready) + self.assertEqual(component.state, ComponentState.READY) + self.assertIn("records=1", component.detail) + + def test_stable_readiness_fails_closed_when_summary_read_fails(self): + class BrokenSpool: + def readiness_summary(self): + raise sqlite3.DatabaseError("cache unavailable") + + readiness = stable_readiness( + self.runtime_config, _Registry(), BrokenSpool(), quota=_Quota() + ) + snapshot = asyncio.run(readiness.snapshot()) + + component = next(item for item in snapshot.components if item.name == "query_cache") + self.assertFalse(snapshot.ready) + self.assertEqual(component.state, ComponentState.NOT_READY) + self.assertIn("DatabaseError", component.detail) diff --git a/tests/test_phaseb_bar_history_bootstrap.py b/tests/test_phaseb_bar_history_bootstrap.py index 672d70a5..725c8566 100644 --- a/tests/test_phaseb_bar_history_bootstrap.py +++ b/tests/test_phaseb_bar_history_bootstrap.py @@ -37,9 +37,14 @@ from qdl.marketdata.v2 import market_data_pb2 from qdl.runtime.stable_bar_edge import ( StableBinanceBarEdge, + _DURABLE_COVERAGE_BATCH_ROWS, _canonical_cache_id, ) from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_capacity import ( + STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, + STABLE_SPOOL_PUBLIC_PARTITION_WINDOW, +) from qdl.runtime.stable_deployment import ( StableAcquisitionPlan, stable_authority_record, @@ -182,6 +187,65 @@ def fetcher(*_args, **kwargs): self.assertEqual(calls[0]["limit"], 1_000) self.assertEqual(calls[0]["end_time"], 1_002 * interval_ms - 1) + def test_binance_history_pages_to_the_declared_v2_maximum(self): + interval_ms = 60_000 + observed_ms = 10_002 * interval_ms + 123 + calls = [] + + def fetcher(*_args, **kwargs): + calls.append(kwargs) + last_open = int(kwargs["end_time"]) - 59_999 + first_open = last_open - (int(kwargs["limit"]) - 1) * interval_ms + return { + "data": [ + _binance_row(first_open + index * interval_ms) + for index in range(int(kwargs["limit"])) + ] + } + + values = fetch_binance_history( + _binance_binding(), + limit=10_000, + now_ms=observed_ms, + fetcher=fetcher, + sleep=lambda _seconds: None, + ) + self.assertEqual(len(values), 10_000) + self.assertEqual([call["limit"] for call in calls], [1_000] * 10) + self.assertEqual(calls[0]["end_time"], 10_002 * interval_ms - 1) + payloads = [json.loads(item.raw_frame_bytes) for item in values] + self.assertEqual(payloads[0]["row"][0], 2 * interval_ms) + self.assertEqual(payloads[-1]["row"][0], 10_001 * interval_ms) + with self.assertRaisesRegex(ValueError, "between 1 and 10000"): + fetch_binance_history( + _binance_binding(), + limit=10_001, + now_ms=observed_ms, + fetcher=fetcher, + sleep=lambda _seconds: None, + ) + + def test_binance_history_rejects_a_repeated_page(self): + interval_ms = 60_000 + observed_ms = 1_002 * interval_ms + 123 + rows = [ + _binance_row((2 + index) * interval_ms) + for index in range(1_000) + ] + with self.assertRaisesRegex(RuntimeError, "exceeds requested end boundary"): + fetch_binance_history( + _binance_binding(), + limit=1_001, + now_ms=observed_ms, + # Ignore the requested cursor but still honour the requested + # page length. The second one-row page is therefore valid in + # shape yet outside its requested end boundary. + fetcher=lambda *_args, **kwargs: { + "data": rows[-int(kwargs["limit"]):] + }, + sleep=lambda _seconds: None, + ) + def test_okx_history_preserves_confirmed_native_rows_and_rejects_partial(self): start = 60_000 records = tuple( @@ -501,6 +565,196 @@ def publish_many(self, _values): finally: spool.close() + def test_checkpoint_history_gap_is_binding_scoped_and_detected_from_cache(self): + """A current checkpoint never certifies a partial durable warmup window.""" + + with tempfile.TemporaryDirectory(prefix="qdl-stable-checkpoint-gap-") as directory: + spool, edge = self._cached_edge(directory, self._NoopPublisher()) + try: + source, _acquisition = next( + pair for pair in edge.history_bindings + if pair[0].instrument.native_symbol == "BTCUSDT" + and pair[0].interval == "1m" + ) + self._cached_final_bar(spool, source, open_ms=120_000) + self.assertEqual( + edge._checkpoint_history_gaps({source.binding_id: 120_000}), + {source.binding_id: 1}, + ) + self._cached_final_bar(spool, source, open_ms=60_000) + self.assertEqual( + edge._checkpoint_history_gaps({source.binding_id: 120_000}), + {}, + ) + finally: + spool.close() + + def test_targeted_history_repair_publishes_only_missing_without_advancing_checkpoint(self): + class Envelope: + def __init__(self, open_ms: int): + self.raw_frame_bytes = json.dumps({"row": _binance_row(open_ms)}).encode() + + class Publisher: + def __init__(self): + self.batches = [] + + def publish_many(self, values): + batch = tuple(values) + self.batches.append(batch) + return tuple(range(len(batch))) + + with tempfile.TemporaryDirectory(prefix="qdl-stable-targeted-repair-") as directory: + publisher = Publisher() + spool, edge = self._cached_edge(directory, publisher) + try: + source, _acquisition = next( + pair for pair in edge.history_bindings + if pair[0].instrument.native_symbol == "BTCUSDT" + and pair[0].interval == "1m" + ) + self._cached_final_bar(spool, source, open_ms=60_000) + edge._last_open_ms[source.binding_id] = 180_000 + with patch( + "qdl.runtime.stable_bar_edge.fetch_binance_history", + return_value=(Envelope(60_000), Envelope(120_000)), + ): + plan = edge.prepare_history_repair( + source.binding_id, + rows=2, + observed_ms=180_000, + ) + self.assertEqual(len(plan.missing_envelopes), 1) + self.assertEqual(edge.history_repair_remaining_rows(plan), 1) + self.assertEqual(edge.apply_history_repair(plan, expected_missing_rows=1), 1) + self.assertEqual(edge._last_open_ms[source.binding_id], 180_000) + self.assertEqual( + [json.loads(item.raw_frame_bytes)["row"][0] for item in publisher.batches[0]], + [120_000], + ) + finally: + spool.close() + + def test_durable_coverage_reads_physical_tail_without_widening_public_limit(self): + with tempfile.TemporaryDirectory(prefix="qdl-stable-physical-tail-") as directory: + spool, edge = self._cached_edge(directory, self._NoopPublisher()) + try: + source, _acquisition = next( + pair for pair in edge.history_bindings + if pair[0].instrument.native_symbol == "BTCUSDT" + and pair[0].interval == "1m" + ) + self._cached_final_bar(spool, source, open_ms=60_000) + payload = spool._connection.execute( + "SELECT payload FROM events WHERE stream = ? AND partition_key = ?", + (source.canonical_stream, source.partition_key), + ).fetchone()[0] + + class Cursor: + def __init__(self): + self.calls = [] + self.rows = [(payload,), (b"\xff",)] + + def fetchmany(self, size): + self.calls.append(size) + rows = self.rows + self.rows = [] + return rows + + class Connection: + def __init__(self): + self.calls = [] + self.cursor = Cursor() + + def execute(self, statement, parameters): + self.calls.append((statement, parameters)) + return self.cursor + + def close(self): + return None + + connection = Connection() + with patch.object(edge, "_assert_canonical_cache_identity"), patch( + "qdl.runtime.stable_bar_edge.sqlite3.connect", + return_value=connection, + ): + covered = edge._durable_final_bar_opens(source, frozenset({60_000})) + + self.assertEqual(covered, frozenset({60_000})) + self.assertEqual( + connection.cursor.calls, [_DURABLE_COVERAGE_BATCH_ROWS] + ) + self.assertEqual(STABLE_SPOOL_PUBLIC_PARTITION_WINDOW, 10_000) + self.assertEqual(STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, 10_064) + self.assertEqual( + connection.calls[0][1], + ( + source.canonical_stream, + source.partition_key, + STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, + ), + ) + finally: + spool.close() + + def test_targeted_history_repair_bound_is_independent_from_runtime_warmup(self): + class Envelope: + def __init__(self, open_ms: int): + self.raw_frame_bytes = json.dumps({"row": _binance_row(open_ms)}).encode() + + with tempfile.TemporaryDirectory(prefix="qdl-stable-targeted-repair-") as directory: + spool, edge = self._cached_edge(directory, self._NoopPublisher()) + try: + source, _acquisition = next( + pair for pair in edge.history_bindings + if pair[0].instrument.native_symbol == "BTCUSDT" + and pair[0].interval == "1m" + ) + self.assertEqual(edge.warmup_rows, 2) + with patch( + "qdl.runtime.stable_bar_edge.fetch_binance_history", + return_value=(Envelope(60_000), Envelope(120_000), Envelope(180_000)), + ): + plan = edge.prepare_history_repair( + source.binding_id, + rows=3, + observed_ms=240_000, + ) + self.assertEqual(len(plan.envelopes), 3) + self.assertEqual(len(plan.missing_envelopes), 3) + finally: + spool.close() + + def test_targeted_history_repair_is_provider_neutral_and_count_fenced(self): + class Envelope: + def __init__(self, open_ms: int): + self.raw_frame_bytes = json.dumps({ + "data": [[str(open_ms), "1", "1", "1", "1", "1", "1", "1", "1"]] + }).encode() + + with tempfile.TemporaryDirectory(prefix="qdl-stable-targeted-repair-") as directory: + spool, edge = self._cached_edge(directory, self._NoopPublisher()) + try: + source, _acquisition = next( + pair for pair in edge.history_okx_bindings + if pair[0].instrument.native_symbol == "BTC-USDT-SWAP" + and pair[0].interval == "1m" + ) + self._cached_final_bar(spool, source, open_ms=60_000) + with patch( + "qdl.runtime.stable_bar_edge.fetch_okx_history", + new_callable=AsyncMock, + return_value=(Envelope(60_000), Envelope(120_000)), + ): + plan = edge.prepare_history_repair( + source.binding_id, + rows=2, + observed_ms=180_000, + ) + with self.assertRaisesRegex(RuntimeError, "missing-row count changed"): + edge.apply_history_repair(plan, expected_missing_rows=0) + finally: + spool.close() + def test_cache_binding_mismatch_fails_closed_before_publish(self): class Envelope: raw_frame_bytes = json.dumps({"row": _binance_row(60_000)}).encode() @@ -617,6 +871,105 @@ def publish_many(self, values): finally: spool.close() + def test_live_cache_generation_change_rebases_then_bootstraps_all_bindings(self): + """A live edge cannot retain watermarks across a cache rebuild.""" + + with tempfile.TemporaryDirectory(prefix="qdl-stable-live-cache-rebase-") as directory: + cache_path = Path(directory) / "canonical-cache.sqlite3" + state_path = Path(directory) / "bar-edge.json" + spool = SQLiteDurableSpool(SpoolConfig( + path=cache_path, + max_records=100, + max_payload_bytes=1_000_000, + max_event_bytes=64_000, + max_storage_bytes=2_000_000, + min_free_disk_bytes=0, + )) + try: + original_cache_id = _canonical_cache_id(cache_path) + edge = StableBinanceBarEdge( + catalog=self.catalog, + acquisition=self.acquisition, + authority=self.authority, + publisher=self._NoopPublisher(), + warmup_rows=2, + state_path=state_path, + canonical_cache_id=original_cache_id, + canonical_cache_path=cache_path, + generation_clock_ns=lambda: 100, + ) + edge._last_open_ms = { + binding_id: 120_000 for binding_id in edge._binding_ids + } + edge._history_bootstrapped = True + edge._retry_attempts = {edge._binding_ids[0]: 1} + edge._next_retry_at = {edge._binding_ids[0]: 200.0} + edge._last_retry_log = {edge._binding_ids[0]: 100.0} + + rebuilt_cache_id = "e" * 32 + with patch( + "qdl.runtime.stable_bar_edge._canonical_cache_id", + return_value=rebuilt_cache_id, + ): + self.assertTrue(edge._rebase_if_canonical_cache_generation_changed()) + + self.assertEqual(edge.canonical_cache_id, rebuilt_cache_id) + self.assertEqual(edge._last_open_ms, {}) + self.assertFalse(edge._history_bootstrapped) + self.assertEqual(edge._retry_attempts, {}) + self.assertEqual(edge._next_retry_at, {}) + self.assertEqual(edge._last_retry_log, {}) + self.assertEqual(edge.connection_generation, 101) + persisted = json.loads(state_path.read_text(encoding="utf-8")) + self.assertEqual(persisted["canonical_cache_id"], rebuilt_cache_id) + self.assertEqual(persisted["last_open_ms"], {}) + + published = [] + + def publish(source, _acquisition, values, *, expected_rows): + self.assertEqual(len(values), expected_rows) + published.append(source.binding_id) + edge._last_open_ms[source.binding_id] = 120_000 + return 0 + + with patch( + "qdl.runtime.stable_bar_edge._canonical_cache_id", + return_value=rebuilt_cache_id, + ), patch.object( + edge, + "_fetch_history", + return_value=(object(), object()), + ), patch.object(edge, "_publish_history", side_effect=publish): + self.assertEqual(edge.bootstrap_history(), 0) + + self.assertEqual(set(published), set(edge._binding_ids)) + self.assertTrue(edge._history_bootstrapped) + + previous_generation = edge.connection_generation + with patch( + "qdl.runtime.stable_bar_edge._canonical_cache_id", + return_value=rebuilt_cache_id, + ): + self.assertFalse(edge._rebase_if_canonical_cache_generation_changed()) + self.assertEqual(edge.connection_generation, previous_generation) + finally: + spool.close() + + def test_live_cache_rebase_fails_closed_when_identity_is_unavailable(self): + with tempfile.TemporaryDirectory(prefix="qdl-stable-live-cache-rebase-") as directory: + spool, edge = self._cached_edge(directory, self._NoopPublisher()) + try: + original_cache_id = edge.canonical_cache_id + with patch( + "qdl.runtime.stable_bar_edge._canonical_cache_id", + side_effect=RuntimeError("stable BAR canonical cache identity is unavailable"), + ), self.assertRaisesRegex(RuntimeError, "identity is unavailable"): + edge._rebase_if_canonical_cache_generation_changed() + self.assertEqual(edge.canonical_cache_id, original_cache_id) + self.assertEqual(edge._last_open_ms, {}) + finally: + spool.close() + def test_cache_short_kafka_ack_does_not_advance_watermark(self): class Envelope: raw_frame_bytes = json.dumps({"row": _binance_row(60_000)}).encode() @@ -686,6 +1039,45 @@ def publish_many(self, values): self.assertEqual(len(edge._last_open_ms), expected_bindings) self.assertTrue(edge._history_bootstrapped) + def test_repair_reuses_live_generation_and_never_updates_writer_checkpoint(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "edge.json" + arguments = dict(catalog=self.catalog, acquisition=self.acquisition, + authority=self.authority, publisher=self._NoopPublisher(), + state_path=path, warmup_rows=2) + writer = StableBinanceBarEdge(**arguments, generation_clock_ns=lambda: 100) + before = path.read_bytes() + repair = StableBinanceBarEdge(**arguments, repair_only=True, + generation_clock_ns=lambda: 999) + self.assertEqual(repair.connection_generation, writer.connection_generation) + self.assertEqual(repair.binance_session_id, writer.binance_session_id) + self.assertEqual(repair.okx_session_id, writer.okx_session_id) + self.assertEqual(path.read_bytes(), before) + repair._assert_repair_writer_current() + for action in (repair._persist_state, repair.run_forever, + repair.bootstrap_history, repair.run_cycle): + with self.assertRaisesRegex(RuntimeError, "repair cannot"): + action() + # A scheduled watermark change is allowed; a new writer is not. + source = next(s for s, _ in writer.history_bindings if s.interval == "1m") + writer._last_open_ms[source.binding_id] = 60_000 + writer._persist_state() + repair._assert_repair_writer_current() + replacement = StableBinanceBarEdge(**arguments, generation_clock_ns=lambda: 101) + self.assertEqual(replacement.connection_generation, 101) + with self.assertRaisesRegex(RuntimeError, "generation or identity changed"): + repair._assert_repair_writer_current() + with self.assertRaisesRegex(RuntimeError, "generation or identity changed"): + repair._apply_history_repair_plan(None, advance_watermark=False) + + def test_repair_fails_without_active_writer_checkpoint(self): + with tempfile.TemporaryDirectory() as directory, self.assertRaisesRegex( + RuntimeError, "active writer checkpoint" + ): + StableBinanceBarEdge(catalog=self.catalog, acquisition=self.acquisition, + authority=self.authority, publisher=self._NoopPublisher(), + state_path=Path(directory) / "missing.json", repair_only=True) + def test_checkpoint_accepts_provider_calendar_anchored_multiday_watermarks(self): with tempfile.TemporaryDirectory(prefix="qdl-stable-calendar-checkpoint-") as raw: state_path = Path(raw) / "bar-edge.json" @@ -832,6 +1224,10 @@ def test_deployed_bootstrap_depth_covers_registered_crypto_bar_demand(self): compose["services"]["binance_bar_edge"]["environment"] ["QDL_STABLE_BAR_WARMUP_ROWS"] ) + catchup = int( + compose["services"]["binance_bar_edge"]["environment"] + ["QDL_STABLE_BAR_MAX_CATCHUP_ROWS"] + ) source_by_id = {item.binding_id: item for item in self.catalog.bindings} history_uids = { source_by_id[item.binding_id].instrument.instrument_uid @@ -853,8 +1249,29 @@ def test_deployed_bootstrap_depth_covers_registered_crypto_bar_demand(self): if item["feed"] == "BAR" and item["instrument_uid"] in history_uids ) - self.assertEqual(max(declared), 1000) + self.assertEqual(max(declared), 10_000) self.assertGreaterEqual(configured, max(declared)) + self.assertGreaterEqual(catchup, max(declared)) + + def test_stable_bar_edge_accepts_the_public_10000_row_bound(self): + edge = StableBinanceBarEdge( + catalog=self.catalog, + acquisition=self.acquisition, + authority=self.authority, + publisher=self._NoopPublisher(), + warmup_rows=10_000, + max_catchup_rows=10_000, + ) + self.assertEqual(edge.warmup_rows, 10_000) + self.assertEqual(edge.max_catchup_rows, 10_000) + with self.assertRaisesRegex(ValueError, "between 1 and 10000"): + StableBinanceBarEdge( + catalog=self.catalog, + acquisition=self.acquisition, + authority=self.authority, + publisher=self._NoopPublisher(), + warmup_rows=10_001, + ) def test_durable_ack_watermark_skips_overlapping_restart_bootstrap(self): class Envelope: diff --git a/tests/test_phaseb_stable_deployment.py b/tests/test_phaseb_stable_deployment.py index 89634466..ca376c06 100644 --- a/tests/test_phaseb_stable_deployment.py +++ b/tests/test_phaseb_stable_deployment.py @@ -30,6 +30,7 @@ SHARED_REALTIME_CORE_GROUP_ID, SHARED_REALTIME_CORE_ID_PREFIX, STABLE_CORE_WORKER_COUNT, + STABLE_CORE_DEDUP_CAPACITY, AuthorityPromotionScope, StableAcquisitionPlan, stable_authority_record, @@ -214,8 +215,11 @@ def test_phase105_external_client_extension_is_additive_and_no_ca_key_is_retaine "issue_client \"${role}\"", "issue_jwt_key \"${role}\"", "spiffe://qdl/paper/monitoring-multivenue-stable", + "spiffe://qdl/paper/trading-system-stable", + "spiffe://qdl/paper/alpha-binance-stable", "spiffe://qdl/paper/alpha-okx-stable", "spiffe://qdl/paper/reference-l2-stable", + "QDL_PHASE105_EXISTING_CLIENT_BUNDLE", "client-ca-bundle.crt", "external-client-ca.key", "rm -f \"${EXTERNAL_CA_KEY}\"", @@ -574,6 +578,7 @@ def test_full_generated_bundle_keeps_every_enabled_binding_on_strict_v2_ingress( } self.assertTrue(core["strict_subscription_scope"]) self.assertEqual(core["raw_topics"], ["md.raw.realtime.v2"]) + self.assertEqual(core["core"]["dedup_capacity"], STABLE_CORE_DEDUP_CAPACITY) self.assertEqual( {item["source_id"] for item in core["core"]["bindings"]}, expected_sources, @@ -1127,12 +1132,61 @@ def test_missing_binding_wrong_provider_kind_and_invalid_primary_authority_fail_ with self.assertRaisesRegex(ValueError, "OKX L2 acquisition"): StableAcquisitionPlan.load(path, catalog=self.catalog) + invalid_hot_l2 = copy.deepcopy(payload) + for item in invalid_hot_l2["bindings"]: + if item["binding_id"] in { + "binance-usdm-ethusdt-book_delta", + "binance-usdm-ethusdt-book_snapshot", + }: + item["l2"]["materialized_snapshot_interval_ms"] = 99 + path.write_text( + yaml.safe_dump(invalid_hot_l2, sort_keys=False), encoding="utf-8" + ) + with self.assertRaisesRegex(ValueError, "materialized snapshot cadence"): + StableAcquisitionPlan.load(path, catalog=self.catalog) + primary = copy.deepcopy(self.authority) primary["mode"] = "RUST_PRIMARY" primary["public_write_allowed"] = True with self.assertRaisesRegex(ValueError, "not an isolated shared Rust"): self.acquisition.core_config(catalog=self.catalog, authority=primary) + def test_hot_l2_materialization_is_core_only_and_keeps_provider_refresh(self): + core = self.acquisition.core_config( + catalog=self.catalog, authority=self.authority + ) + from qdl.runtime.execution_l2 import execution_l2_materialization_plan + + execution_l2 = execution_l2_materialization_plan( + demand_path=ROOT / "config/v2/stable-crypto-demand.yaml", + catalog=self.catalog, + acquisition=self.acquisition, + ) + hot = { + item["source_id"]: item["l2"] + for item in core["core"]["bindings"] + if item.get("l2", {}).get("materialized_snapshot_interval_ms") == 1_000 + } + self.assertEqual(set(hot), set(execution_l2.source_ids)) + self.assertEqual(len(hot), 10) + self.assertTrue(all( + item["depth_per_side"] == 100 + and item["snapshot_refresh_seconds"] == 30 + and item["materialized_snapshot_interval_ms"] == 1_000 + for item in hot.values() + )) + ingestors = self.acquisition.native_ingestor_configs( + catalog=self.catalog, authority=self.authority + ) + for config in ingestors.values(): + for binding in config["bindings"]: + if ( + binding["subscription_id"] in execution_l2.source_ids + and binding["feed"] == "BOOK" + ): + self.assertNotIn("materialized_snapshot_interval_ms", binding["l2"]) + self.assertEqual(binding["l2"]["snapshot_refresh_seconds"], 30) + def test_generated_primary_authority_builds_every_shared_runtime_role(self): primary = stable_authority_record( rust_image_digest="d" * 64, @@ -1220,6 +1274,24 @@ def test_compose_is_isolated_bounded_nonroot_and_has_no_v1_route(self): self.assertTrue( all(str(port).startswith("127.0.0.1:") for port in services[name]["ports"]) ) + for name in ("query_v2_1", "query_v2_2"): + with self.subTest(query_healthcheck=name): + healthcheck = services[name]["healthcheck"] + command = " ".join(healthcheck["test"]) + self.assertIn("https://localhost:8200/health/ready", command) + self.assertIn("/stable-certs/query/ca.crt", command) + self.assertEqual(healthcheck["interval"], "5s") + self.assertEqual(healthcheck["timeout"], "3s") + self.assertEqual(healthcheck["retries"], 20) + for name in ("stream_v2_active", "stream_v2_passive"): + with self.subTest(stream_healthcheck=name): + healthcheck = services[name]["healthcheck"] + command = " ".join(healthcheck["test"]) + self.assertIn("https://localhost:8200/health/dependencies", command) + self.assertIn("/stable-certs/stream/ca.crt", command) + self.assertEqual(healthcheck["interval"], "5s") + self.assertEqual(healthcheck["timeout"], "3s") + self.assertEqual(healthcheck["retries"], 20) projector_names = ("projector_v2", "projector_v2_2", "projector_v2_3") for name in projector_names: self.assertNotIn("ports", services[name]) @@ -1324,6 +1396,12 @@ def test_compose_is_isolated_bounded_nonroot_and_has_no_v1_route(self): services[name]["networks"]["stable_consumer"]["aliases"], [alias], ) + for name in ("stream_v2_active", "stream_v2_passive"): + alias = ingress_aliases[name] + self.assertEqual( + services[name]["networks"]["stable_internal"]["aliases"], + [alias], + ) for name in ("query_v2_1", "query_v2_2"): with self.subTest(pass_through_query_role=name): self.assertEqual( diff --git a/tests/test_phaseb_stable_edge.py b/tests/test_phaseb_stable_edge.py index 812e45ca..6e097460 100644 --- a/tests/test_phaseb_stable_edge.py +++ b/tests/test_phaseb_stable_edge.py @@ -2,9 +2,11 @@ import base64 import hashlib +import hmac import json import os import tempfile +from types import SimpleNamespace from urllib.parse import urlsplit import unittest from datetime import datetime, timezone @@ -56,8 +58,12 @@ from qdl.raw.capture import bind_capture_context, capture_exact_frame from qdl.replay import GapFreeHandoff, SignedHandoffCursorCodec from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_capacity import ( + STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, +) from qdl.runtime.stable import ( StableRuntimeConfig, + stable_request_bounds, stable_grpc_server_credentials, stable_uvicorn_tls, ) @@ -74,7 +80,7 @@ ) from qdl.runtime.session_liveness import StableSessionLivenessReader from qdl.stream import DurableStreamGateway -from qdl.transport import DurableEvent, SQLiteDurableSpool, SpoolConfig +from qdl.transport import BackpressureRequired, DurableEvent, SQLiteDurableSpool, SpoolConfig from qdl.transport.kafka_projector import KafkaProjectorRecord from qdl.warmup import WarmupSpecification, WarmupTimeRange @@ -410,6 +416,7 @@ def setUp(self): max_payload_bytes=8 * 1024 * 1024, max_storage_bytes=16 * 1024 * 1024, min_free_disk_bytes=0, + max_partition_records=STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, )) def tearDown(self): @@ -592,6 +599,7 @@ def event_at(index: int): gap_spool = SQLiteDurableSpool(SpoolConfig( path=Path(directory) / "market-tail-gap.sqlite3", min_free_disk_bytes=0, + max_partition_records=STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, )) try: for index in (6, 7, 9, 10, 11): @@ -611,6 +619,7 @@ def event_at(index: int): invalid_spool = SQLiteDurableSpool(SpoolConfig( path=Path(directory) / "market-tail-lineage.sqlite3", min_free_disk_bytes=0, + max_partition_records=STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, )) try: for index in range(8, 13): @@ -631,6 +640,97 @@ def event_at(index: int): finally: invalid_spool.close() + def test_public_bar_warmup_scans_physical_tail_before_market_selection(self): + binding = next( + item + for item in self.catalog.bindings + if item.binding_id == "binance-usdm-btcusdt-bar-1m" + ) + template = _stable_event( + self.catalog, "binance_usdm_rest_bar.json", binding.binding_id + ) + minute_ns = 60 * 1_000_000_000 + + def stored(index: int, offset: int): + event = type(template)() + event.CopyFrom(template) + event.event_id = hashlib.sha256( + f"phase-b-physical-tail-{index}".encode() + ).digest()[:16] + event.raw_capture_id = hashlib.sha256( + f"phase-b-physical-tail-raw-{index}".encode() + ).digest()[:16] + event.bar.open_time_ns = template.bar.open_time_ns + index * minute_ns + event.bar.close_time_ns = template.bar.close_time_ns + index * minute_ns + event.source_event_time_ns = event.bar.close_time_ns + event.received_at_ns = event.bar.close_time_ns + 1 + event.normalized_at_ns = event.received_at_ns + 1 + event.published_at_ns = event.received_at_ns + 2 + event.source_sequence = f"phase-b-physical-tail-{index}" + event.partition_sequence = offset + event.correlation_id = f"phase-b-physical-tail-{index}" + return SimpleNamespace( + event=SimpleNamespace( + payload=event.SerializeToString(), event_id=bytes(event.event_id) + ), + cursor=SimpleNamespace( + stream=binding.canonical_stream, + partition_key=binding.partition_key, + offset=offset, + ), + ) + + # The first logical records simulate authentic late repairs. Once the + # cache has grown, a public-size append-tail scan would evict precisely + # these recent market-time bars and manufacture a false warmup gap. + repaired_indices = tuple(range(9_900, 9_964)) + repaired_set = set(repaired_indices) + remaining_indices = tuple( + index + for index in range(-64, 10_000) + if index not in repaired_set + ) + rows = tuple( + stored(index, offset) + for offset, index in enumerate( + (*repaired_indices, *remaining_indices), start=1 + ) + ) + self.assertEqual(len(rows), STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW) + + class PhysicalTailSpool: + def __init__(self, values): + self.values = values + self.limits = [] + + def read_tail(self, *, stream, partition_key, limit): + self.limits.append((stream, partition_key, limit)) + return tuple(self.values[-limit:]) + + spool = PhysicalTailSpool(rows) + backend = StableSpoolQueryBackend( + spool, + self.catalog, + schema_digest="e" * 64, + clock_ns=lambda: template.bar.close_time_ns + 10_001 * minute_ns, + ) + history = backend.history(_requirement(binding, warmup=700)) + self.assertIsNotNone(history) + self.assertEqual(history.coverage.value, "FULL") + self.assertEqual( + [item.payload["open_time_ns"] for item in history.items], + [ + template.bar.open_time_ns + index * minute_ns + for index in range(9_300, 10_000) + ], + ) + self.assertFalse(backend.latest(_requirement(binding, warmup=700)).quality.gap_open) + self.assertTrue(spool.limits) + self.assertTrue(all( + limit == STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW + for _stream, _partition, limit in spool.limits + )) + def test_vn_time_range_accepts_lunch_break_but_rejects_missing_session_bar(self): binding = next( item @@ -696,6 +796,7 @@ def event_at(open_ns, index): incomplete_spool = SQLiteDurableSpool(SpoolConfig( path=Path(directory) / "incomplete.sqlite3", min_free_disk_bytes=0, + max_partition_records=STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW, )) try: _append(incomplete_spool, self.catalog, event_at(start_ns, 1)) @@ -750,7 +851,10 @@ def tracked_read_tail(**kwargs): self.assertIsNotNone(backend.latest(_requirement(trade))) self.assertIsNotNone(backend.history(_requirement(trade, warmup=1))) self.assertIsNotNone(backend.latest(_requirement(bar))) - self.assertEqual(observed_limits, [1, 1, 10_000]) + self.assertEqual( + observed_limits, + [1, 1, STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW], + ) def test_latest_uses_newest_tail_after_partition_exceeds_query_window(self): binding = next( @@ -1311,6 +1415,50 @@ async def sleep(delay): self.assertEqual(sleeps, [0.25]) self.assertEqual(active, [brokers[0], None, brokers[1], None]) + async def test_run_once_defers_a_polled_record_at_the_batch_byte_bound(self): + class Record: + def __init__(self, payload): + self.payload = payload + + class PollingBroker(_Broker): + def __init__(self, records): + super().__init__() + self.records = list(records) + + def poll(self, timeout_seconds): + del timeout_seconds + return self.records.pop(0) if self.records else None + + first = Record(b"aaa") + second = Record(b"bbb") + broker = PollingBroker((first, second)) + engine = StableProjectorEngine( + broker=broker, + spool=self.spool, + catalog=self.catalog, + canonical_topic="qdl.stable.canonical.phase-b.v2", + raw_topics=(), + sink=LocalStableCanonicalSink(self.gateway, self.spool), + projector=StableCompatibilityProjector(self.catalog), + target=InMemoryStableProjectionTarget(), + max_pending_records=10, + max_pending_bytes=1024, + max_batch_records=4, + max_batch_bytes=4, + ) + accepted = [] + + async def accept_many(records): + accepted.append(tuple(records)) + + engine.accept_many = accept_many + self.assertTrue(await engine.run_once(timeout_seconds=0.01)) + self.assertEqual(accepted, [(first,)]) + self.assertIs(engine._deferred_record, second) + self.assertTrue(await engine.run_once(timeout_seconds=0.01)) + self.assertEqual(accepted, [(first,), (second,)]) + self.assertIsNone(engine._deferred_record) + async def test_canonical_before_raw_waits_and_checkpoints_after_all_downstreams(self): binding, raw, event = _stable_pair( self.catalog, "binance_usdm_trade.json", "binance-usdm-btcusdt-trade" @@ -2142,6 +2290,51 @@ async def test_signed_http_sink_commits_to_active_gateway_idempotently(self): await sink.close() await client.aclose() + async def test_canonical_ingest_maps_shared_spool_capacity_to_typed_503(self): + binding, raw, event = _stable_pair( + self.catalog, "binance_usdm_trade.json", "binance-usdm-btcusdt-trade" + ) + raw_topic, _canonical_topic, raw_record, canonical_record = _broker_records( + binding, raw, event + ) + app = FastAPI() + secret = b"phase-b-stable-capacity-secret-32" + install_stable_canonical_ingest( + app, gateway=self.gateway, catalog=self.catalog, + spool=self.spool, secret=secret, + ) + body = json.dumps({ + "schema": "qdl.v2.stable-canonical-ingest.v1", + "batch_id": "00000000-0000-4000-8000-000000000001", + "events": [{ + "canonical": base64.b64encode(canonical_record.payload).decode("ascii"), + "raw_stream": raw_topic, + "raw_event_id": raw_record.event_id.hex(), + "raw_provider_envelope": base64.b64encode(raw_record.payload).decode("ascii"), + }], + }, sort_keys=True, separators=(",", ":")).encode() + signature = "sha256=" + hmac.new(secret, body, hashlib.sha256).hexdigest() + client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) + try: + with patch.object( + self.gateway, + "publish_many", + side_effect=BackpressureRequired("bridge max_records exhausted"), + ): + response = await client.post( + "/internal/v2/canonical/events", content=body, + headers={"X-QDL-Stable-Signature": signature}, + ) + self.assertEqual(response.status_code, 503) + self.assertEqual( + response.json(), + {"detail": "stable canonical cache capacity temporarily unavailable"}, + ) + finally: + await client.aclose() + async def test_signed_http_sink_chunks_by_exact_request_bytes(self): durable = tuple( DurableEvent( @@ -2335,9 +2528,13 @@ def test_query_role_is_isolated_and_projector_dependencies_fail_closed(self): root / "state" / "session-liveness", ) values["QDL_STABLE_REQUEST_DEADLINE_SECONDS"] = "90" + configured = StableRuntimeConfig.from_environment("query_v2", values) + self.assertEqual(configured.request_deadline_seconds, 90.0) + bounds = stable_request_bounds(configured) + self.assertEqual(bounds.request_deadline_seconds, 90.0) + self.assertEqual(bounds.max_request_bytes, configured.max_request_bytes) self.assertEqual( - StableRuntimeConfig.from_environment("query_v2", values).request_deadline_seconds, - 90.0, + bounds.max_concurrent_requests, configured.max_concurrent_requests ) values["QDL_STABLE_REQUEST_DEADLINE_SECONDS"] = "121" with self.assertRaisesRegex(ValueError, "request deadline"): @@ -2361,12 +2558,28 @@ def test_query_role_is_isolated_and_projector_dependencies_fail_closed(self): values.update({ "QDL_STABLE_MAX_PENDING_RECORDS": "2048", "QDL_STABLE_MAX_PENDING_BYTES": "33554432", + "QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS": "512", + "QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES": "8388608", }) bounded_projector = StableRuntimeConfig.from_environment( "projector_v2", values ) self.assertEqual(bounded_projector.max_pending_records, 2048) self.assertEqual(bounded_projector.max_pending_bytes, 33_554_432) + self.assertEqual(bounded_projector.projector_max_batch_records, 512) + self.assertEqual(bounded_projector.projector_max_batch_bytes, 8_388_608) + values["QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS"] = "2049" + with self.assertRaisesRegex(ValueError, "projector batch bound"): + StableRuntimeConfig.from_environment("projector_v2", values) + values["QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS"] = "512" + values["QDL_STABLE_MAX_PENDING_RECORDS"] = "64" + with self.assertRaisesRegex(ValueError, "pending records"): + StableRuntimeConfig.from_environment("projector_v2", values) + values["QDL_STABLE_MAX_PENDING_RECORDS"] = "2048" + values["QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES"] = "33554433" + with self.assertRaisesRegex(ValueError, "batch byte bound"): + StableRuntimeConfig.from_environment("projector_v2", values) + values["QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES"] = "8388608" authority_path = Path(values["QDL_STABLE_RUNTIME_DIR"]) / "authority.json" primary = json.loads(authority_path.read_text(encoding="utf-8")) primary.update({"mode": "RUST_PRIMARY", "revision": 2}) diff --git a/tests/test_phaseb_stable_rebuild.py b/tests/test_phaseb_stable_rebuild.py index 94712976..f7e75b95 100644 --- a/tests/test_phaseb_stable_rebuild.py +++ b/tests/test_phaseb_stable_rebuild.py @@ -8,11 +8,13 @@ from unittest.mock import patch from scripts.rebuild_v2_stable_projection_cache import ( + BAR_EDGE_SERVICES, CACHE_FILES, CANONICAL_TOPIC, CONFIRM_TOKEN, EXPECTED_CANONICAL_PARTITIONS, MAX_ACCEPTED_LAG, + MAX_ACCEPTED_PARTITION_LAG, PROJECT_NAME, PROJECTOR_GROUP, PROJECTOR_SERVICES, @@ -60,9 +62,18 @@ def test_plan_is_exact_isolated_and_v1_safe(self): plan["lag_gate"]["max_total_records"], MAX_ACCEPTED_LAG, ) + self.assertEqual( + plan["lag_gate"]["max_per_partition_records"], + MAX_ACCEPTED_PARTITION_LAG, + ) self.assertEqual( plan["start_order"], - [list(STREAM_SERVICES), list(PROJECTOR_SERVICES), list(QUERY_SERVICES)], + [ + list(STREAM_SERVICES), + list(PROJECTOR_SERVICES), + list(BAR_EDGE_SERVICES), + list(QUERY_SERVICES), + ], ) self.assertFalse(plan["touches_v1"]) self.assertFalse(plan["apply"]) @@ -272,24 +283,42 @@ def test_lag_parser_requires_real_canonical_partitions(self): stable-projector-v1 md.canonical.v2 1 20 20 0 - - - stable-projector-v1 another.topic 2 0 99 99 - - - """ - self.assertEqual(parse_canonical_lag(output), (2, 2)) + self.assertEqual(parse_canonical_lag(output), (2, 2, 2)) with self.assertRaisesRegex(RuntimeError, "no partitions"): parse_canonical_lag("GROUP TOPIC PARTITION") + with self.assertRaisesRegex(RuntimeError, "repeats a partition"): + parse_canonical_lag( + "stable-projector-v1 md.canonical.v2 0 10 12 2 - - -\n" + "stable-projector-v1 md.canonical.v2 0 12 14 2 - - -\n" + ) - def test_lag_gate_requires_all_partitions_and_fixed_bound(self): + def test_lag_gate_requires_all_partitions_and_two_dimensional_bound(self): self.assertTrue( lag_sample_acceptable( - MAX_ACCEPTED_LAG, EXPECTED_CANONICAL_PARTITIONS + MAX_ACCEPTED_LAG, + EXPECTED_CANONICAL_PARTITIONS, + MAX_ACCEPTED_PARTITION_LAG, + ) + ) + self.assertFalse( + lag_sample_acceptable( + MAX_ACCEPTED_LAG + 1, + EXPECTED_CANONICAL_PARTITIONS, + MAX_ACCEPTED_PARTITION_LAG, ) ) self.assertFalse( lag_sample_acceptable( - MAX_ACCEPTED_LAG + 1, EXPECTED_CANONICAL_PARTITIONS + 0, + EXPECTED_CANONICAL_PARTITIONS - 1, + 0, ) ) self.assertFalse( lag_sample_acceptable( - 0, EXPECTED_CANONICAL_PARTITIONS - 1 + MAX_ACCEPTED_LAG, + EXPECTED_CANONICAL_PARTITIONS, + MAX_ACCEPTED_PARTITION_LAG + 1, ) ) @@ -350,6 +379,7 @@ def test_catchup_gets_full_timeout_after_projectors_start(self): env = Path("/tmp/stable.env") cache_sizes = iter(("0\n", "2\n")) observed = {} + starts = [] def fake_compose(_env, *arguments, **_kwargs): if arguments[:4] == ("ps", "--services", "--status", "running"): @@ -368,6 +398,9 @@ def capture_lag(_env, deadline): def capture_projector(_env, deadline): observed["projector"] = deadline + def capture_start(_env, *services): + starts.append(services) + with ( patch("scripts.rebuild_v2_stable_projection_cache._validate_project"), patch( @@ -383,7 +416,7 @@ def capture_projector(_env, deadline): "scripts.rebuild_v2_stable_projection_cache._stable_client_ssl_context", return_value=None, ), - patch("scripts.rebuild_v2_stable_projection_cache._start_services"), + patch("scripts.rebuild_v2_stable_projection_cache._start_services", capture_start), patch("scripts.rebuild_v2_stable_projection_cache._wait_http", capture_http), patch("scripts.rebuild_v2_stable_projection_cache._wait_bounded_lag", capture_lag), patch("scripts.rebuild_v2_stable_projection_cache._wait_projector_ready", capture_projector), @@ -396,6 +429,10 @@ def capture_projector(_env, deadline): self.assertEqual(observed["lag"], 170.0) self.assertEqual(observed["projector"], 170.0) self.assertEqual(observed["http"][2:], [170.0, 170.0]) + self.assertEqual( + starts, + [STREAM_SERVICES, PROJECTOR_SERVICES, BAR_EDGE_SERVICES, QUERY_SERVICES], + ) if __name__ == "__main__": diff --git a/tests/test_phaseb_stable_release.py b/tests/test_phaseb_stable_release.py index 10abd33d..7052b7a2 100644 --- a/tests/test_phaseb_stable_release.py +++ b/tests/test_phaseb_stable_release.py @@ -3,7 +3,10 @@ import copy import json import tempfile -import tomllib +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib import unittest from dataclasses import replace from pathlib import Path @@ -177,6 +180,9 @@ def close(self): ) self.assertFalse(broker._consumer.config["enable.auto.commit"]) self.assertEqual(broker._consumer.config["isolation.level"], "read_committed") + self.assertEqual(broker._consumer.config["queued.max.messages.kbytes"], 16 * 1024) + self.assertEqual(broker._consumer.config["fetch.max.bytes"], 8 * 1024 * 1024) + self.assertEqual(broker._consumer.config["max.partition.fetch.bytes"], 2 * 1024 * 1024) broker.close() self.assertFalse(broker.ping()) @@ -304,14 +310,14 @@ def record(offset, *, partition=0, epoch=1): class StableReleaseVersionContractTests(unittest.TestCase): - def test_package_sdk_and_openapi_are_exactly_2_0_0(self): + def test_service_openapi_and_sdk_versions_are_explicit(self): package = tomllib.loads((ROOT / "pyproject.toml").read_text(encoding="utf-8")) snapshot = json.loads( (ROOT / "contracts/v2/openapi.snapshot.json").read_text(encoding="utf-8") ) generated = build_openapi() self.assertEqual(package["project"]["version"], "2.0.0") - self.assertEqual(qdl_sdk.__version__, "2.0.0") + self.assertEqual(qdl_sdk.__version__, "2.0.1") self.assertEqual(generated["info"]["version"], "2.0.0") self.assertEqual(snapshot, generated) # ``reference:batch`` is a governed V2 public path in the checked-in diff --git a/tests/test_phasec36_reference_l2_rollout.py b/tests/test_phasec36_reference_l2_rollout.py index ab357370..9617e3f4 100644 --- a/tests/test_phasec36_reference_l2_rollout.py +++ b/tests/test_phasec36_reference_l2_rollout.py @@ -88,12 +88,14 @@ def _inputs(self, root: Path) -> dict[str, object]: "stable-alpha-binance-rs256-v1": self._key("binance"), "stable-monitoring-rs256-v1": self._key("monitoring-jwt"), "stable-alpha-okx-rs256-v1": self._key("alpha-okx-jwt"), + "stable-reference-l2-rs256-v1": self._key("reference-l2-jwt"), } subjects = { "stable-trading-system-rs256-v1": "spiffe://qdl/paper/trading-system-stable", "stable-alpha-binance-rs256-v1": "spiffe://qdl/paper/alpha-binance-stable", "stable-monitoring-rs256-v1": "spiffe://qdl/paper/monitoring-multivenue-stable", "stable-alpha-okx-rs256-v1": "spiffe://qdl/paper/alpha-okx-stable", + "stable-reference-l2-rs256-v1": "spiffe://qdl/paper/reference-l2-stable", } base_env = root / "base.env" self._write_env(base_env, { diff --git a/tests/test_python310_compat.py b/tests/test_python310_compat.py new file mode 100644 index 00000000..9c147599 --- /dev/null +++ b/tests/test_python310_compat.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import importlib +from enum import auto +import unittest + +from qdl._compat import StrEnum + + +class Python310CompatTests(unittest.TestCase): + def test_string_enum_keeps_wire_value_behavior(self) -> None: + class Sample(StrEnum): + ACTIVE = "ACTIVE" + + self.assertIsInstance(Sample.ACTIVE, str) + self.assertEqual(Sample.ACTIVE, "ACTIVE") + self.assertEqual(str(Sample.ACTIVE), "ACTIVE") + self.assertEqual(Sample.ACTIVE.value, "ACTIVE") + + def test_auto_value_matches_stdlib_str_enum(self) -> None: + class Sample(StrEnum): + LIVE = auto() + + self.assertIsInstance(Sample.LIVE, str) + self.assertEqual(Sample.LIVE.value, "live") + self.assertEqual(str(Sample.LIVE), "live") + + def test_internal_string_enum_modules_import_through_shared_compat(self) -> None: + modules = ( + "qdl.canary.phase9", + "qdl.canary.phase92", + "qdl.canary.phase93", + "qdl.certification.phase103_consumer_acceptance", + "qdl.certification.prerequisites", + "qdl.consumer.manifest", + "qdl.consumer.realtime_route", + "qdl.consumer.universal_release", + "qdl.data_quality.engine", + "qdl.data_quality.source_authority", + "qdl.domain.lifecycle", + "qdl.domain.quantity", + "qdl.query.contracts", + "qdl.query.entitlement", + "qdl.query.lifecycle", + "qdl.security.data_plane", + "qdl.warmup.contracts", + ) + for name in modules: + with self.subTest(module=name): + importlib.import_module(name) diff --git a/tests/test_qdl_sdk_read_reconnect.py b/tests/test_qdl_sdk_read_reconnect.py new file mode 100644 index 00000000..6ffbea53 --- /dev/null +++ b/tests/test_qdl_sdk_read_reconnect.py @@ -0,0 +1,70 @@ +import unittest +from unittest.mock import patch + +import httpx + +from qdl_sdk.transport import RestQueryTransport + + +class ReadReconnectTests(unittest.IsolatedAsyncioTestCase): + async def test_read_methods_retry_closed_socket_once_with_remaining_budget(self): + for method, path in (("get", "/v2/feeds/x/status"), + ("post", "/v2/market-data/warmup:batch"), + ("post", "/v2/market-data/reference:batch")): + calls = [] + + def handle(request): + calls.append(request) + if len(calls) == 1: + raise httpx.RemoteProtocolError("peer closed", request=request) + return httpx.Response(200, json={"status": "OK"}) + + async with httpx.AsyncClient(base_url="https://qdl.test", transport=httpx.MockTransport(handle)) as client: + transport = RestQueryTransport("https://qdl.test", client=client, timeout_seconds=10) + with patch("qdl_sdk.transport.time.monotonic", side_effect=[100.0, 102.0]): + response = await transport._read_request(method, path) + self.assertEqual(response.status_code, 200) + self.assertEqual(len(calls), 2) + self.assertEqual(calls[0].method, calls[1].method) + self.assertEqual(calls[0].url, calls[1].url) + self.assertEqual(calls[1].extensions["timeout"]["read"], 8.0) + + async def test_permanent_socket_failure_is_bounded_and_timeout_is_not_retried(self): + for error_type, count in ((httpx.RemoteProtocolError, 2), (httpx.ReadTimeout, 1)): + calls = [] + + def handle(request): + calls.append(request) + raise error_type("unavailable", request=request) + + async with httpx.AsyncClient(base_url="https://qdl.test", transport=httpx.MockTransport(handle)) as client: + transport = RestQueryTransport("https://qdl.test", client=client) + with self.assertRaises(error_type): + await transport._read_request("get", "/v2/feeds/x/status") + self.assertEqual(len(calls), count) + + async def test_exhausted_budget_and_http_errors_are_not_retried(self): + calls = [] + + def closed(request): + calls.append(request) + raise httpx.RemoteProtocolError("closed", request=request) + + async with httpx.AsyncClient(base_url="https://qdl.test", transport=httpx.MockTransport(closed)) as client: + transport = RestQueryTransport("https://qdl.test", client=client, timeout_seconds=10) + with patch("qdl_sdk.transport.time.monotonic", side_effect=[100.0, 111.0]): + with self.assertRaises(httpx.RemoteProtocolError): + await transport._read_request("get", "/v2/feeds/x/status") + self.assertEqual(len(calls), 1) + for code in (401, 403, 429, 503): + calls = [] + + def handle(request): + calls.append(request) + return httpx.Response(code, json={"code": "DATA_STALE"}) + + async with httpx.AsyncClient(base_url="https://qdl.test", transport=httpx.MockTransport(handle)) as client: + transport = RestQueryTransport("https://qdl.test", client=client) + result = await transport._read_request("get", "/v2/feeds/x/status") + self.assertEqual(result.status_code, code) + self.assertEqual(len(calls), 1) diff --git a/tests/test_qdl_sdk_release.py b/tests/test_qdl_sdk_release.py index 5d5031dc..75686dc5 100644 --- a/tests/test_qdl_sdk_release.py +++ b/tests/test_qdl_sdk_release.py @@ -28,11 +28,12 @@ def test_wheel_is_reproducible_self_contained_and_recorded(self): self.assertIn("qdl/query/v2/query_pb2.py", names) self.assertIn("qdl/marketdata/v2/market_data_pb2.py", names) self.assertFalse(any(name.startswith("qdl/api_v2/") for name in names)) - record_name = "qdl_sdk-2.0.0.dist-info/RECORD" + record_name = f"qdl_sdk-{first_result['version']}.dist-info/RECORD" rows = list(csv.reader(io.StringIO(archive.read(record_name).decode()))) self.assertEqual({row[0] for row in rows}, names) release = json.loads(Path(first_result["manifest"]).read_text()) self.assertFalse(release["contains_service_internals"]) + self.assertEqual(release["version"], "2.0.1") self.assertEqual(len(release["generated_contract_digest"]), 64) sbom = json.loads(Path(first_result["sbom"]).read_text()) self.assertEqual(sbom["bomFormat"], "CycloneDX") diff --git a/tests/test_qdl_sdk_stream_projection.py b/tests/test_qdl_sdk_stream_projection.py index d51c2113..9450cef6 100644 --- a/tests/test_qdl_sdk_stream_projection.py +++ b/tests/test_qdl_sdk_stream_projection.py @@ -409,6 +409,30 @@ def test_gap_and_stale_execution_events_fail_closed(self): ) self.assertEqual(stale.exception.code, "DATA_STALE") + def test_stale_execution_replay_is_non_executable_but_gap_stays_strict(self): + replay = market_data_view_from_stream( + StreamEvent(11, "signed", envelope(Feed.QUOTE)), + template=template(Feed.QUOTE), + requirement=self.requirement(Feed.QUOTE), + now_ns=NOW + 2_000_000_000, + replay_only=True, + ) + self.assertEqual(replay.quality.state, "STALE") + self.assertEqual(replay.quality.event_recency_state, "STALE") + self.assertFalse(replay.quality.execution_eligible) + + gapped = envelope(Feed.QUOTE) + gapped.quality_flags.append(common_pb2.QUALITY_FLAG_SEQUENCE_GAP_BEFORE) + with self.assertRaises(ContinuityError) as error: + market_data_view_from_stream( + StreamEvent(12, "signed", gapped), + template=template(Feed.QUOTE), + requirement=self.requirement(Feed.QUOTE), + now_ns=NOW + 2_000_000_000, + replay_only=True, + ) + self.assertEqual(error.exception.code, "OPEN_SEQUENCE_GAP") + def test_quiet_connected_trade_is_observable_but_never_execution_eligible(self): requirement = DataRequirement( instrument_uid="uid-1", @@ -472,6 +496,109 @@ def test_query_sdk_observes_quiet_trade_but_rejects_disconnected_session(self): warmup=False, ) + def test_query_sdk_observes_only_governed_quiet_book_delta(self): + requirement = DataRequirement( + instrument_uid="uid-1", + feed=Feed.BOOK_DELTA, + consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", + max_freshness_ms=1_000, + event_recency_policy=StalePolicy.OBSERVE, + max_session_liveness_ms=45_000, + ) + data = template(Feed.BOOK_DELTA).model_dump(mode="json") + data["quality"].update( + { + "freshness_ms": 2_000, + "event_recency_state": "STALE", + "provider_session_state": "LIVE", + "provider_session_liveness_ms": 1, + "execution_eligible": False, + } + ) + accepted = _validate_query_payload( + requirement, + {"request_id": "quiet-book-delta", "data": data}, + warmup=False, + ) + self.assertFalse(accepted.data.quality.execution_eligible) + + blocked_policy = DataRequirement( + instrument_uid="uid-1", + feed=Feed.BOOK_DELTA, + consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", + max_freshness_ms=1_000, + event_recency_policy=StalePolicy.BLOCK, + max_session_liveness_ms=45_000, + ) + with self.assertRaises(ContinuityError): + _validate_query_payload( + blocked_policy, + {"request_id": "blocked-book-delta", "data": data}, + warmup=False, + ) + + with self.assertRaisesRegex(ValueError, "provider session SLA"): + DataRequirement( + instrument_uid="uid-1", + feed=Feed.BOOK_DELTA, + consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", + max_freshness_ms=1_000, + event_recency_policy=StalePolicy.OBSERVE, + ) + + data["quality"]["provider_session_state"] = "DISCONNECTED" + with self.assertRaisesRegex(ContinuityError, "provider session"): + _validate_query_payload( + requirement, + {"request_id": "disconnected-book-delta", "data": data}, + warmup=False, + ) + + data["quality"].update({"provider_session_state": "LIVE", "gap_open": True}) + with self.assertRaisesRegex(ContinuityError, "open gap"): + _validate_query_payload( + requirement, + {"request_id": "gapped-book-delta", "data": data}, + warmup=False, + ) + + data["quality"].update({"gap_open": False, "complete": False}) + with self.assertRaisesRegex(ContinuityError, "execution eligible"): + _validate_query_payload( + requirement, + {"request_id": "partial-book-delta", "data": data}, + warmup=False, + ) + + book_snapshot = template(Feed.BOOK_SNAPSHOT).model_dump(mode="json") + book_snapshot["quality"].update( + { + "freshness_ms": 2_000, + "event_recency_state": "STALE", + "provider_session_state": "LIVE", + "provider_session_liveness_ms": 1, + "execution_eligible": False, + } + ) + snapshot_requirement = DataRequirement( + instrument_uid="uid-1", + feed=Feed.BOOK_SNAPSHOT, + consumer_grade=Grade.EXECUTION, + source_policy_id="crypto_primary_v2", + max_freshness_ms=1_000, + event_recency_policy=StalePolicy.OBSERVE, + max_session_liveness_ms=45_000, + ) + with self.assertRaisesRegex(ContinuityError, "execution eligible"): + _validate_query_payload( + snapshot_requirement, + {"request_id": "quiet-book-snapshot", "data": book_snapshot}, + warmup=False, + ) + def test_execution_book_requires_verified_generation(self): unverified = envelope(Feed.BOOK_SNAPSHOT) unverified.book_snapshot.sequence_verified = False diff --git a/tests/test_reference_l2_consumer_acceptance.py b/tests/test_reference_l2_consumer_acceptance.py index e8443355..302b4d9b 100644 --- a/tests/test_reference_l2_consumer_acceptance.py +++ b/tests/test_reference_l2_consumer_acceptance.py @@ -127,6 +127,51 @@ def test_reference_acceptance_batches_preserve_every_product_and_isolate_basis(s self.assertEqual(len(isolated), 5) self.assertTrue(all(len(batch) == 1 for batch in isolated)) + def test_reference_acceptance_batches_isolate_strict_mark_snapshots(self): + marks = tuple( + item for item in self.scope.references + if item.requirement.feed is FeedType.MARK_INDEX_PRICE + ) + self.assertGreaterEqual(len(marks), 3) + strict_marks = tuple( + replace( + item, + requirement=replace(item.requirement, max_freshness_ms=2_000), + sdk_requirement=item.sdk_requirement.model_copy( + update={"max_freshness_ms": 2_000} + ), + ) + for item in marks[:2] + ) + loose_mark = replace( + marks[2], + requirement=replace(marks[2].requirement, max_freshness_ms=2_001), + sdk_requirement=marks[2].sdk_requirement.model_copy( + update={"max_freshness_ms": 2_001} + ), + ) + ordinary = next( + item for item in self.scope.references + if item.requirement.feed is FeedType.FUNDING_RATE + ) + + batches = reference_acceptance_batches((*strict_marks, loose_mark, ordinary)) + flattened = tuple(item for batch in batches for item in batch) + + self.assertEqual( + tuple(batch[0].identity for batch in batches[:2]), + tuple(item.identity for item in strict_marks), + ) + self.assertTrue(all(len(batch) == 1 for batch in batches[:2])) + self.assertEqual( + {item.identity for item in flattened}, + {item.identity for item in (*strict_marks, loose_mark, ordinary)}, + ) + self.assertEqual( + {item.identity for item in batches[-1]}, + {loose_mark.identity, ordinary.identity}, + ) + def test_acceptance_transport_timeout_keeps_provider_deadline_bounded(self): self.assertEqual(acceptance_transport_timeout_seconds(60.0), 75.0) self.assertEqual(acceptance_transport_timeout_seconds(5.0), 20.0) diff --git a/tests/test_refresh_v2_l2_core_runtime.py b/tests/test_refresh_v2_l2_core_runtime.py index 77088f26..0be1c48f 100644 --- a/tests/test_refresh_v2_l2_core_runtime.py +++ b/tests/test_refresh_v2_l2_core_runtime.py @@ -3,6 +3,7 @@ import json import tempfile import unittest +from functools import lru_cache from pathlib import Path from qdl.runtime.stable_catalog import StableSourceCatalog @@ -11,9 +12,9 @@ stable_authority_record, write_stable_runtime_bundle, ) +from qdl.runtime.execution_l2 import execution_l2_materialization_plan from scripts.refresh_v2_l2_core_runtime import ( CORE_FILES, - DECLARED_ADDITIVE_BOOK_SOURCE_IDS, refresh, ) @@ -24,6 +25,17 @@ RUST_IMAGE = "sha256:" + "a" * 64 +@lru_cache(maxsize=1) +def _execution_l2_source_ids() -> frozenset[str]: + catalog = StableSourceCatalog.load(CATALOG) + acquisition = StableAcquisitionPlan.load(ACQUISITION, catalog=catalog) + return frozenset(execution_l2_materialization_plan( + demand_path=ROOT / "config/v2/stable-crypto-demand.yaml", + catalog=catalog, + acquisition=acquisition, + ).source_ids) + + class L2CoreRuntimeRefreshTests(unittest.TestCase): def _authority(self) -> dict[str, object]: return stable_authority_record( @@ -54,7 +66,7 @@ def _runtime(self, root: Path) -> Path: payload["core"]["bindings"] = [ item for item in payload["core"]["bindings"] - if item["source_id"] not in DECLARED_ADDITIVE_BOOK_SOURCE_IDS + if item["source_id"] not in _execution_l2_source_ids() ] for item in payload["core"]["bindings"]: item["instrument_catalog_revision"] = 7 @@ -78,12 +90,17 @@ def test_dry_run_adds_exact_declared_l2_scope_without_mutation(self): self.assertTrue(result["authority_bytes_preserved"]) for file_name in CORE_FILES: item = result["files"][file_name] - self.assertEqual(item["before_binding_count"], 176) - self.assertEqual(item["after_binding_count"], 182) - self.assertEqual(item["catalog_revision_updated_binding_count"], 176) + self.assertEqual( + item["after_binding_count"] - item["before_binding_count"], + len(_execution_l2_source_ids()), + ) + self.assertEqual( + item["catalog_revision_updated_binding_count"], + item["before_binding_count"], + ) self.assertEqual(item["before_catalog_revisions"], [7]) self.assertEqual(item["after_catalog_revisions"], [8]) - self.assertEqual(set(item["added_book_source_ids"]), DECLARED_ADDITIVE_BOOK_SOURCE_IDS) + self.assertEqual(set(item["added_book_source_ids"]), _execution_l2_source_ids()) self.assertEqual({name: (runtime / name).read_bytes() for name in CORE_FILES}, before) def test_apply_preserves_existing_bindings_authority_and_modes(self): @@ -128,7 +145,10 @@ def test_apply_preserves_existing_bindings_authority_and_modes(self): updated_by_id[key]["instrument_catalog_revision"] == 8 for key in active_by_id )) - self.assertEqual(set(updated_by_id) - set(active_by_id), DECLARED_ADDITIVE_BOOK_SOURCE_IDS) + self.assertEqual( + set(updated_by_id) - set(active_by_id), + _execution_l2_source_ids(), + ) def test_dry_run_converges_stale_metadata_after_l2_was_already_added(self): with tempfile.TemporaryDirectory() as raw: @@ -150,7 +170,7 @@ def test_dry_run_converges_stale_metadata_after_l2_was_already_added(self): existing_ids = {item["source_id"] for item in active["core"]["bindings"]} active["core"]["bindings"].extend( item for item in expected["core"]["bindings"] - if item["source_id"] in DECLARED_ADDITIVE_BOOK_SOURCE_IDS + if item["source_id"] in _execution_l2_source_ids() and item["source_id"] not in existing_ids ) path.write_text(json.dumps(active, indent=2, sort_keys=True) + "\n", encoding="utf-8") @@ -162,11 +182,13 @@ def test_dry_run_converges_stale_metadata_after_l2_was_already_added(self): ) for file_name in CORE_FILES: item = result["files"][file_name] - self.assertEqual(item["before_binding_count"], 182) - self.assertEqual(item["after_binding_count"], 182) - self.assertEqual(item["catalog_revision_updated_binding_count"], 176) + self.assertEqual(item["before_binding_count"], item["after_binding_count"]) + self.assertEqual( + item["catalog_revision_updated_binding_count"], + item["before_binding_count"] - len(_execution_l2_source_ids()), + ) self.assertEqual(item["added_book_source_ids"], []) - self.assertEqual(set(item["declared_book_source_ids"]), DECLARED_ADDITIVE_BOOK_SOURCE_IDS) + self.assertEqual(set(item["declared_book_source_ids"]), _execution_l2_source_ids()) def test_rejects_existing_semantic_drift(self): with tempfile.TemporaryDirectory() as raw: diff --git a/tests/test_refresh_v2_rust_core_runtime.py b/tests/test_refresh_v2_rust_core_runtime.py index eca86ac1..d19b15cf 100644 --- a/tests/test_refresh_v2_rust_core_runtime.py +++ b/tests/test_refresh_v2_rust_core_runtime.py @@ -3,15 +3,21 @@ import json import tempfile import unittest +from functools import lru_cache from pathlib import Path from qdl.runtime.stable_catalog import StableSourceCatalog from qdl.runtime.stable_deployment import ( + STABLE_CORE_DEDUP_CAPACITY, StableAcquisitionPlan, stable_authority_record, write_stable_runtime_bundle, ) -from scripts.refresh_v2_rust_core_runtime import CORE_FILES, refresh +from qdl.runtime.execution_l2 import execution_l2_materialization_plan +from scripts.refresh_v2_rust_core_runtime import ( + CORE_FILES, + refresh, +) ROOT = Path(__file__).resolve().parents[1] @@ -21,6 +27,17 @@ NEW_IMAGE = "sha256:" + "b" * 64 +@lru_cache(maxsize=1) +def _execution_l2_source_ids() -> frozenset[str]: + catalog = StableSourceCatalog.load(CATALOG) + acquisition = StableAcquisitionPlan.load(ACQUISITION, catalog=catalog) + return frozenset(execution_l2_materialization_plan( + demand_path=ROOT / "config/v2/stable-crypto-demand.yaml", + catalog=catalog, + acquisition=acquisition, + ).source_ids) + + class RustCoreRuntimeRefreshTests(unittest.TestCase): def _authority(self) -> dict[str, object]: return stable_authority_record( @@ -44,9 +61,10 @@ def _active_runtime(self, root: Path) -> tuple[Path, Path, bytes]: authority_bytes = (runtime / "authority.json").read_bytes() for name in CORE_FILES: payload = json.loads((runtime / name).read_text()) + payload["core"]["dedup_capacity"] = 1_000_000 for binding in payload["core"]["bindings"]: - if binding.get("l2") is not None: - binding["l2"].pop("snapshot_refresh_seconds", None) + if binding.get("source_id") in _execution_l2_source_ids(): + binding["l2"].pop("materialized_snapshot_interval_ms", None) (runtime / name).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") (runtime / "unrelated.json").write_text('{"untouched":true}\n') environment = root / "rollout.env" @@ -69,13 +87,24 @@ def _refresh(self, root: Path, *, apply: bool) -> dict[str, object]: state_root=root / "state", ) - def test_dry_run_proves_only_l2_cadence_changes(self): + def test_dry_run_proves_only_declared_hot_l2_materialization_changes(self): with tempfile.TemporaryDirectory() as raw: result = self._refresh(Path(raw), apply=False) self.assertEqual(result["status"], "DRY_RUN") self.assertTrue(result["authority_bytes_preserved"]) self.assertEqual(len(result["changes"]), 3) self.assertTrue(all(item["l2_source_ids"] for item in result["changes"])) + self.assertTrue(all( + set(item["execution_l2_source_ids"]) == _execution_l2_source_ids() + for item in result["changes"] + )) + self.assertTrue( + all( + item["dedup_capacity"] + == {"before": 1_000_000, "after": STABLE_CORE_DEDUP_CAPACITY} + for item in result["changes"] + ) + ) self.assertEqual(result["production_mutations"], 0) def test_apply_keeps_authority_and_unrelated_runtime_file(self): @@ -97,13 +126,43 @@ def test_apply_keeps_authority_and_unrelated_runtime_file(self): self.assertIn(NEW_IMAGE, environment.read_text()) for name in CORE_FILES: payload = json.loads((runtime / name).read_text()) + self.assertEqual(payload["core"]["dedup_capacity"], STABLE_CORE_DEDUP_CAPACITY) self.assertTrue(all( binding["l2"].get("snapshot_refresh_seconds") == 30 for binding in payload["core"]["bindings"] if binding.get("l2") is not None )) + self.assertEqual( + { + binding["source_id"] + for binding in payload["core"]["bindings"] + if binding.get("l2", {}).get("materialized_snapshot_interval_ms") == 1000 + }, + _execution_l2_source_ids(), + ) self.assertTrue((root / "state" / "refresh" / "rollback" / name).is_file()) + def test_config_only_refresh_reuses_selected_immutable_image(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + runtime, environment, _ = self._active_runtime(root) + environment.write_text( + "QDL_CONFIG_REVISION=phasec36-reference-l2-r13\n" + f"QDL_STABLE_RUST_IMAGE={NEW_IMAGE}\n" + "QDL_STABLE_RUNTIME_DIR=/runtime\n" + ) + result = refresh( + runtime_dir=runtime, + rollout_env=environment, + active_rust_image=NEW_IMAGE, + new_rust_image=NEW_IMAGE, + output_dir=None, + apply=False, + state_root=root / "state", + ) + self.assertFalse(result["image_selector_changed"]) + self.assertEqual(result["production_mutations"], 0) + def test_fails_closed_when_non_l2_binding_changes(self): with tempfile.TemporaryDirectory() as raw: root = Path(raw) @@ -123,6 +182,24 @@ def test_fails_closed_when_non_l2_binding_changes(self): state_root=root / "state", ) + def test_fails_closed_when_dedup_capacity_is_not_an_approved_predecessor(self): + with tempfile.TemporaryDirectory() as raw: + root = Path(raw) + runtime, environment, _ = self._active_runtime(root) + payload = json.loads((runtime / "core.json").read_text()) + payload["core"]["dedup_capacity"] = 100_001 + (runtime / "core.json").write_text(json.dumps(payload)) + with self.assertRaisesRegex(ValueError, "bounded dedup transition"): + refresh( + runtime_dir=runtime, + rollout_env=environment, + active_rust_image=OLD_IMAGE, + new_rust_image=NEW_IMAGE, + output_dir=None, + apply=False, + state_root=root / "state", + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_release_session_observations.py b/tests/test_release_session_observations.py new file mode 100644 index 00000000..4f8b5bc0 --- /dev/null +++ b/tests/test_release_session_observations.py @@ -0,0 +1,78 @@ +from dataclasses import replace +import unittest + +from qdl.consumer.release import ReleaseRouteObservation, v2_observation_is_current +from qdl.certification.phase105_release import parse_release_observations +from qdl.certification.phase105_release_observations import compact_view_quality +from qdl.query.contracts import ConsumerGrade, DataRequirement, FeedType, StalePolicy +from types import SimpleNamespace + + +class ReleaseSessionObservationTests(unittest.TestCase): + def requirement(self, feed=FeedType.TRADE, policy=StalePolicy.OBSERVE): + return DataRequirement(instrument_uid="test-only", feed=feed, + consumer_grade=ConsumerGrade.EXECUTION, source_policy_id="test-policy", + max_freshness_ms=3000, max_session_liveness_ms=45000, + event_recency_policy=policy) + + def observation(self): + return ReleaseRouteObservation(consumer_id="test-only", requirement_key="test-only", + route="V2_PRIMARY", reason="V2_READY", v2_source_age_ms=12000, + v2_receive_age_ms=12050, v2_gap_open=False, v1_source_age_ms=None, + v1_receive_age_ms=None, consumer_lag=0, cpu_millicores=1, rss_bytes=1, + v2_quality_state="LIVE", v2_session_state="LIVE", v2_session_liveness_ms=1000, + v2_complete=True, v2_execution_eligible=False) + + def test_quiet_session_matrix_preserves_actual_age_and_price_safety(self): + for venue in ("BINANCE", "OKX"): + for symbol in ("BTC", "ETH", "SOL", "DOGE", "BNB"): + for feed in (FeedType.TRADE, FeedType.BOOK_DELTA): + with self.subTest(venue=venue, symbol=symbol, feed=feed): + observation = replace(self.observation(), requirement_key=f"{venue}:{symbol}:{feed.value}") + self.assertTrue(v2_observation_is_current(self.requirement(feed), observation)) + self.assertEqual(observation.v2_source_age_ms, 12000) + self.assertFalse(observation.v2_execution_eligible) + self.assertEqual(parse_release_observations([observation.public_record()]), (observation,)) + for policy in (StalePolicy.BLOCK, StalePolicy.PAUSE): + self.assertFalse(v2_observation_is_current(self.requirement(feed, policy), observation)) + for feed in (FeedType.QUOTE, FeedType.BOOK_SNAPSHOT, FeedType.MARK_INDEX_PRICE): + self.assertFalse(v2_observation_is_current(self.requirement(feed), self.observation())) + + def test_incomplete_gap_expired_disconnected_or_unproven_session_stays_blocked(self): + for fields in ( + {"v2_gap_open": True}, {"v2_complete": False}, {"v2_quality_state": "STALE"}, + {"v2_session_state": "DISCONNECTED"}, {"v2_session_state": "UNKNOWN"}, + {"v2_session_state": "STALE"}, {"v2_session_state": "NOT_APPLICABLE"}, + {"v2_session_liveness_ms": None}, {"v2_session_liveness_ms": 45001}, + {"v2_execution_eligible": True}, + {"v2_quality_state": None, "v2_session_state": None, + "v2_session_liveness_ms": None, "v2_complete": None, "v2_execution_eligible": None}, + ): + with self.subTest(fields=fields): + self.assertFalse(v2_observation_is_current(self.requirement(), replace(self.observation(), **fields))) + + def test_fresh_price_still_requires_typed_live_complete_session(self): + observation = replace(self.observation(), v2_source_age_ms=100, + v2_receive_age_ms=200, v2_execution_eligible=True) + requirement = self.requirement(FeedType.QUOTE, StalePolicy.BLOCK) + self.assertTrue(v2_observation_is_current(requirement, observation)) + self.assertFalse(v2_observation_is_current(requirement, + replace(observation, v2_session_state="DISCONNECTED"))) + + def test_compact_quality_does_not_discard_session_or_make_quiet_data_executable(self): + quality = SimpleNamespace(freshness_ms=12000, gap_open=False, state="LIVE", + provider_session_state="LIVE", provider_session_liveness_ms=900, + complete=True, execution_eligible=False) + evidence = compact_view_quality(SimpleNamespace(quality=quality, received_at_ns=1000000), + observed_at_ns=13000000) + self.assertEqual(evidence["source_age_ms"], 12000) + self.assertEqual(evidence["provider_session_liveness_ms"], 900) + self.assertFalse(evidence["execution_eligible"]) + + def test_typed_evidence_rejects_bad_types_and_extra_fields(self): + for fields in ({"v2_session_liveness_ms": True}, {"v2_complete": 1}, + {"v2_quality_state": "UNVALIDATED"}, {"v2_session_liveness_ms": -1}): + with self.subTest(fields=fields), self.assertRaises(ValueError): + replace(self.observation(), **fields) + with self.assertRaises(ValueError): + parse_release_observations([{**self.observation().public_record(), "extra": True}]) diff --git a/tests/test_routed_query_backend.py b/tests/test_routed_query_backend.py index ec13cf63..96993b24 100644 --- a/tests/test_routed_query_backend.py +++ b/tests/test_routed_query_backend.py @@ -134,6 +134,25 @@ def _requirement(self, uid: str, **overrides) -> DataRequirement: def test_it_satisfies_the_backend_protocol(self): self.assertIsInstance(self.backend, MarketDataQueryBackend) + def test_local_lane_never_includes_provider_fallback(self): + for venue in ("BINANCE", "OKX"): + binding = next(b for b in self.catalog.bindings + if b.instrument.identity.venue == venue and b.feed is FeedType.BAR) + requirement = self._requirement( + binding.instrument.instrument_uid, + source_policy_id=binding.source_policy_id, interval=binding.interval, + recovery=RecoveryPolicy.SNAPSHOT_AND_REPLAY, + ) + self.assertTrue(self.backend.warmup_is_local(requirement)) + from dataclasses import replace + self.assertFalse(self.backend.warmup_is_local( + replace(requirement, recovery=RecoveryPolicy.FRESH_SNAPSHOT) + )) + self.assertFalse(self.backend.warmup_is_local(self._requirement(self.unbound_uid))) + self.assertTrue(RoutedQueryBackend(self.spool).warmup_is_local( + self._requirement(BINANCE_BTC) + )) + def test_a_bound_instrument_always_uses_the_authoritative_spool(self): # Declaring FRESH_SNAPSHOT must never downgrade a covered consumer. requirement = self._requirement(BINANCE_BTC) diff --git a/upgrade/evidence/releases/v2.0.12/RELEASE_NOTES.md b/upgrade/evidence/releases/v2.0.12/RELEASE_NOTES.md new file mode 100644 index 00000000..2b7f120d --- /dev/null +++ b/upgrade/evidence/releases/v2.0.12/RELEASE_NOTES.md @@ -0,0 +1,73 @@ +# Quant Data Layer v2.0.12 + +## Certified Scope + +Real-provider, no-order certification passed all five release gates for the +declared Binance USD-M and OKX Swap manifests. Rust remains canonical authority; +Python exposes stable V2 query/stream and SDK contracts. V1 is fallback only +where explicitly allowed. This package is ready for a release PR from dev to +main; this file alone does not assert GitHub publication. + +- 299/299 V2 consumer products: monitoring4, Trading-System paper adapter60, + Binance alpha SDK125, OKX alpha SDK110. +- 234 durable products and65 bounded on-demand reference products. +- BTC, ETH, SOL, DOGE and BNB across both venues;140 physical final-BAR + bindings,10 execution books, declared venue-specific intervals/metrics. +- TRADE, QUOTE, final BAR, BOOK_SNAPSHOT/BOOK_DELTA, MARK_INDEX_PRICE, + funding, open interest, contract metadata, and the declared Binance + long/short, taker-flow and basis products. Unsupported OKX metrics are not + substituted with Binance data. +- Zero blocked crypto routes, zero active fallback at certification, no + resource-budget violations. Seven allowed fallback routes passed the + V2 -> V1 -> V2 drill;292 disallowed fallback routes remained fail-closed. + +## Corrections + +- Keep stable history under bounded count/byte/disk retention instead of + deleting sparse BARs solely by commit age. All140 retained BAR windows were + verified complete with no holes, duplicates or sequence-gap flags. +- Repair genuine provider BARs using the active writer generation; never + fence normal ingestion with an unrelated repair generation. +- Separate local canonical cache reads from provider REST admission quotas. +- Retry one prematurely closed read-only HTTP connection within its original + deadline. Do not retry execution, authorization or provider-throttling errors. +- Align certification with typed live-session semantics, bounded reference + deadlines and immutable BAR overlap at close boundaries. Separate hot + closing batches from history; retain strict source and receive freshness. +- Update an inactive optional QUIC dependency with no change to the selected + Rust production dependency graph. Final scanned image has zero fixable + HIGH/CRITICAL findings in the recorded scan. + +## Evidence + +Full C2: opening883.031s (including identity quota pacing), observation300.100s, +closing13.401s. Opening duration is not endpoint latency. Zero order actions, +zero direct-provider connections from test clients, and client/cursor cleanup +completed. All44 protected services were unchanged. + +Source regression, contract, SDK, Rust and CI evidence is linked from the +[main implementation journal](../../../../DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md). +Exact scope, source/harness hashes and runtime digests are in +[scope-evidence.json](scope-evidence.json); the machine-readable result is +[certificate.json](certificate.json). + +## Runtime And Rollback + +- Python8 roles: `qdl-v2-python:2.0.12-35a7cd8`, + `sha256:1c1392bf636dc40c67cc73a2e5ea5e8d17f4e53ca4ecb8c62ac387be4262045a`. +- Rust core/ingestion: `qdl-v2-rust:2.0.12-d536098`, + `sha256:36a822c0ef61fb122dbf8fa12221cff27ad6a863976424be1407cd345f4dce65`. +- Runtime revision: `phasec36-reference-l2-r14`; no new topology. +- Immediate rollback retains query image `cca6355c` and other Python-role + image `46a04c1e`, with exact existing runtime/state/TLS mounts. +- V1, Kafka offsets/topology, Redis/SQLite generations, Trading System and + alpha/order paths remain protected. No offset reset or data deletion. + +## Boundaries + +This is a single-host, real-provider data-plane certificate for the listed +consumer manifests, not broker-order execution, independent HA/DR or every +possible future product. Four VN routes remain explicitly V1-primary pending +their separate market-hours certification. Per-consumer manifest identity, +entitlement, freshness and risk policy still apply. No trading alpha was +started by this release closure. diff --git a/upgrade/evidence/releases/v2.0.12/certificate.json b/upgrade/evidence/releases/v2.0.12/certificate.json new file mode 100644 index 00000000..6f01235c --- /dev/null +++ b/upgrade/evidence/releases/v2.0.12/certificate.json @@ -0,0 +1,56 @@ +{ + "gates": [ + { + "evidence": "99022ac411eb7f49ceb88b3105996a7a7042284db158a76ec72a23e79d44783e", + "gate_id": "release_route_readiness", + "status": "PASS" + }, + { + "evidence": "a305cd316e4df8d904903baec7198e2033d820838ae89252712af38ab621fd44", + "gate_id": "v1_fallback_provenance", + "status": "PASS" + }, + { + "evidence": "bb163a4e3333429248a7f260538be89692898d7d6be3caf77ed3960b8dc3d5f0", + "gate_id": "runtime_handoff", + "status": "PASS" + }, + { + "evidence": "6aaba9a12080dbde67292b4efac26620c911e8eab91ce27a0c3644490de0e1be", + "gate_id": "consumer_v2_primary", + "status": "PASS" + }, + { + "evidence": "63b59ceed967bdaac0d942eacd958b63454dd3f5d97715295117e846de9b012a", + "gate_id": "v1_fallback_return", + "status": "PASS" + } + ], + "input_sha256": { + "consumer_acceptance": "6aaba9a12080dbde67292b4efac26620c911e8eab91ce27a0c3644490de0e1be", + "fallback_drill": "63b59ceed967bdaac0d942eacd958b63454dd3f5d97715295117e846de9b012a", + "observations": "7afbfa1488cd00082ff5b8c8e16f77c5c6bfd65644ab0d402373ba53a2dc47af", + "runtime_handoff": "bb163a4e3333429248a7f260538be89692898d7d6be3caf77ed3960b8dc3d5f0", + "v1_provenance": "a305cd316e4df8d904903baec7198e2033d820838ae89252712af38ab621fd44" + }, + "readiness": { + "blocked_count": 0, + "budget_violations": [], + "fallback_count": 0, + "fallback_rate": 0.0, + "max_consumer_lag": 11, + "max_cpu_millicores": 117, + "max_rss_bytes": 215113728, + "product_count": 303, + "ready": true, + "route_plan_sha256": "99022ac411eb7f49ceb88b3105996a7a7042284db158a76ec72a23e79d44783e", + "status": "READY", + "v1_primary_count": 4, + "v2_primary_count": 299 + }, + "release_route_plan_sha256": "99022ac411eb7f49ceb88b3105996a7a7042284db158a76ec72a23e79d44783e", + "schema": "qdl.phase105.stable-release-certification.v1", + "status": "PASS", + "v2_python_image": "sha256:1c1392bf636dc40c67cc73a2e5ea5e8d17f4e53ca4ecb8c62ac387be4262045a", + "v2_rust_image": "sha256:36a822c0ef61fb122dbf8fa12221cff27ad6a863976424be1407cd345f4dce65" +} diff --git a/upgrade/evidence/releases/v2.0.12/scope-evidence.json b/upgrade/evidence/releases/v2.0.12/scope-evidence.json new file mode 100644 index 00000000..961513a0 --- /dev/null +++ b/upgrade/evidence/releases/v2.0.12/scope-evidence.json @@ -0,0 +1,181 @@ +{ + "authority": { + "approved_by": "BobbyAxerol", + "candidate_image_digest": "sha256:3056cf849d4d767f19431af92b944698b4dbef15c044942831619d296f8cd156", + "capability_manifest_digest": "81347a88866a54ed99cfbf25979d8e5912c98084f97f8a997a608dc9e03af6bf", + "contract_digest": "100966435a30367dfa20dd8f9d94af15fb0c27f684aeedbd3ac9511bb757d13b", + "effective_at_ns": 1787636881636650743, + "legacy_write_allowed": false, + "mode": "RUST_PRIMARY", + "partition_plan_digest": "521c9c3ad307e94fb9decf1043401f7cccdf2d13dd93e200bdaffb3645abece0", + "public_write_allowed": false, + "revision": 1, + "schema": "qdl.authority-record.v1", + "slice_id": "qdl-v2-shared-realtime-primary" + }, + "c2_harness_source_commit": "e24a42c9efa2a8653bc80123b55ea73f86c31b63", + "c2_receipt_sha256": "f5a8e479d3019eea360c4848a00090c053268bcc135fce45ec6109b9751d5f8c", + "certificate_sha256": "0b67916ef677f7ee099de510a1731948b2eea85e876663ff99cf7d67e4bc1b2e", + "certificate_source_sha256": { + "qdl/certification/phase105_release.py": "26974d50bbcc54300c97ad6ba8439ce9accc14a7f6df7fbdc4bd5cc3de2b7898", + "qdl/certification/phase105_release_observations.py": "60e2bf5e274708ca3a70bb3b199bb3a6cf0f428c5b92138333c0a7de40252939", + "qdl/consumer/release.py": "75777f000998d20e92ad3c28debeb672440e64026063ed8e1d3fd5f6ca348d36", + "scripts/phase105_consumer_v2_identity_acceptance.py": "26ca97f87cce8fd0147cbfa4f4ee9dbcc9abdcf1e0469fc04a36b8cafdca8068" + }, + "classification": "single-host real-provider no-order data-plane certification", + "closing_seconds": 13.401, + "current_reads_capture": { + "captured_at_ms": 1788592094845, + "cpu_millicores": 117, + "rss_bytes": 215113728 + }, + "current_reads_sha256": "bf24384d57cbab2050da2162c024988cf487f039a753daffc8c84df4ff883a8a", + "cursor_directory_removed": true, + "direct_provider_connections": 0, + "durable_product_count": 234, + "exclusions": [ + "VN market-hours cutover", + "broker order execution", + "independent failure-domain HA or DR", + "unlisted consumer/product bindings" + ], + "harness_sha256": { + "phase103_consumer_receipt_acceptance.py": "5e3777002c94d30f2d350294c2c7fca57d164f55dcb4d7167b82324fbe942df8", + "phase105_consumer_v2_identity_acceptance.py": "45f7c9b8f7b49271acb33652c061c253c1c73642dd7158e0a544180dc3143d5b", + "phasec36_reference_l2_consumer_acceptance.py": "6a3535e9b06fa19f5403c6e11cc84ae6f09b4d134b6ed584533ab4196bceeb1d" + }, + "observation_seconds": 300.1, + "opening_seconds": 883.031, + "order_actions": 0, + "product_count": 299, + "products_by_consumer": { + "alpha.binance.paper.stable": 125, + "alpha.okx.paper.stable": 110, + "monitoring.multivenue.stable": 4, + "trading-system.paper.stable": 60 + }, + "products_by_feed": { + "BAR": 150, + "BASIS": 5, + "BOOK_DELTA": 20, + "BOOK_SNAPSHOT": 20, + "CONTRACT_METADATA": 10, + "FUNDING_RATE": 10, + "LONG_SHORT_RATIO": 5, + "MARK_INDEX_PRICE": 20, + "OPEN_INTEREST": 10, + "QUOTE": 20, + "TAKER_FLOW": 5, + "TRADE": 24 + }, + "protected_changes": [], + "release": "v2.0.12", + "release_capture": { + "captured_at_ms": 1788590216588, + "cpu_millicores": 69, + "rss_bytes": 256983040 + }, + "route_contract": { + "blocked_fallback_declared": 292, + "blocked_route_count": 292, + "blocked_v1_requests": 0, + "route_selection_probe_only": true, + "v1_fallback_declared": 7, + "v1_fallback_observed": true + }, + "route_plan_sha256": "99022ac411eb7f49ceb88b3105996a7a7042284db158a76ec72a23e79d44783e", + "schema": "qdl.release.scope-evidence.v1", + "scope_sha256": "bc7c15b3b3e0014a0b2a1302870378f1af8fe090fa1b1b89e43a032408f8b4a0", + "serving_roles": { + "binance_bar_edge": { + "container_id": "e0be7b8e6b6d58b5bc5aa5fa8bad230205aebd1e6e3bf1e123c4ea17870419ed", + "image": "sha256:1c1392bf636dc40c67cc73a2e5ea5e8d17f4e53ca4ecb8c62ac387be4262045a", + "oom": false, + "restarts": 0, + "running": true, + "started_at": "2026-09-05T05:17:16.31633881Z" + }, + "projector_v2": { + "container_id": "b4fb2fbbc73415b47b3717f1cf575d1a589b8c5ae13d42fc04b862fbaf135dc3", + "image": "sha256:1c1392bf636dc40c67cc73a2e5ea5e8d17f4e53ca4ecb8c62ac387be4262045a", + "oom": false, + "restarts": 0, + "running": true, + "started_at": "2026-09-05T05:17:11.934729697Z" + }, + "projector_v2_2": { + "container_id": "c1d8fb1ea78924561314d5bdf0af28cdbddb817198f065b2b8e96f17a61cd0ba", + "image": "sha256:1c1392bf636dc40c67cc73a2e5ea5e8d17f4e53ca4ecb8c62ac387be4262045a", + "oom": false, + "restarts": 0, + "running": true, + "started_at": "2026-09-05T05:17:13.334779269Z" + }, + "projector_v2_3": { + "container_id": "ffc4296b80a52fd23ccd3af877eba48e891b5e725d739e2d5bebf0491e958c9d", + "image": "sha256:1c1392bf636dc40c67cc73a2e5ea5e8d17f4e53ca4ecb8c62ac387be4262045a", + "oom": false, + "restarts": 0, + "running": true, + "started_at": "2026-09-05T05:17:14.911796261Z" + }, + "query_v2_1": { + "container_id": "78a90b6baba1651e9a172f56dbd6c323e8bf409cd147b97d903b155db64abf06", + "image": "sha256:1c1392bf636dc40c67cc73a2e5ea5e8d17f4e53ca4ecb8c62ac387be4262045a", + "oom": false, + "restarts": 0, + "running": true, + "started_at": "2026-09-05T05:16:07.572804427Z" + }, + "query_v2_2": { + "container_id": "11c1dc340c7987984c996dd5b1b01036325b55ab2d7d5653f8587abd631f452c", + "image": "sha256:1c1392bf636dc40c67cc73a2e5ea5e8d17f4e53ca4ecb8c62ac387be4262045a", + "oom": false, + "restarts": 0, + "running": true, + "started_at": "2026-09-05T05:16:30.939675884Z" + }, + "rust_core": { + "container_id": "c82bf6e89433edce277ff4a5836baa477fb956e118cbbaa0e7b795cc5d9bedda", + "image": "sha256:36a822c0ef61fb122dbf8fa12221cff27ad6a863976424be1407cd345f4dce65", + "oom": false, + "restarts": 0, + "running": true, + "started_at": "2026-09-05T01:05:33.196156486Z" + }, + "rust_core_2": { + "container_id": "2e88cfad09c883a387eefad62b6d6a60735c7380aa21de324754940b08094031", + "image": "sha256:36a822c0ef61fb122dbf8fa12221cff27ad6a863976424be1407cd345f4dce65", + "oom": false, + "restarts": 0, + "running": true, + "started_at": "2026-09-04T15:10:11.552899391Z" + }, + "rust_core_3": { + "container_id": "1a752e11308c41cdf87f40468f6a130326715b00467e2a41032d2eaef2a8ab29", + "image": "sha256:36a822c0ef61fb122dbf8fa12221cff27ad6a863976424be1407cd345f4dce65", + "oom": false, + "restarts": 0, + "running": true, + "started_at": "2026-09-04T15:10:19.633622632Z" + }, + "stream_v2_active": { + "container_id": "35f66776e077d65075640f6a4372f6dd5ed74e465102983c772b5cfc25c53b98", + "image": "sha256:1c1392bf636dc40c67cc73a2e5ea5e8d17f4e53ca4ecb8c62ac387be4262045a", + "oom": false, + "restarts": 0, + "running": true, + "started_at": "2026-09-05T05:16:44.582293595Z" + }, + "stream_v2_passive": { + "container_id": "2204caa72efc6e5e7b4e93cf7763ed51d1b76c6b296bc24b89d3b086b64147a1", + "image": "sha256:1c1392bf636dc40c67cc73a2e5ea5e8d17f4e53ca4ecb8c62ac387be4262045a", + "oom": false, + "restarts": 0, + "running": true, + "started_at": "2026-09-05T05:16:58.470129778Z" + } + }, + "source_commit": "f0118d8874fd8806124a0070d2a45b8968b554c2", + "status": "PASS" +} diff --git a/upgrade/quant-data-layer-fund-grade-upgrade-architecture.md b/upgrade/quant-data-layer-fund-grade-upgrade-architecture.md index 166fae1a..ee4c2a8f 100644 --- a/upgrade/quant-data-layer-fund-grade-upgrade-architecture.md +++ b/upgrade/quant-data-layer-fund-grade-upgrade-architecture.md @@ -1027,6 +1027,16 @@ Với derived bar có thể partition theo `instrument_uid + interval`. Retention là config theo environment/data class, không hardcode trong adapter. +Stable canonical projection clarification (2026-09-05): its advertised warmup +is a bounded record window, not a 24-hour commit-age window. The shared spool +opts into `retain_partition_windows` only with a positive per-partition bound +(currently 10,064 physical records, at most 10,000 public rows). Global record, +payload/disk limits and signed-cursor expiry remain enforced. Consumer ACK or +age maintenance must not punch holes into this window. Generic/raw spools keep +their existing age policy. An explicit historical repair reuses the active BAR +writer checkpoint generation, validates it before publication and never writes +that checkpoint; writer restart fences an in-flight repair normally. + ### 11.4 Raw and canonical transaction Hai deployment mode: @@ -1477,6 +1487,13 @@ Mọi switch có cooldown/hysteresis để tránh flapping và có audit record. ## 15. Historical storage and short warmup architecture +Canonical-cache read admission (2026-09-05): provider REST token budgets apply +only to reads that may call that provider. A backend-declared local-only +canonical warmup uses bounded concurrency (8), singleflight and request +deadlines without a vendor token wait. A missing binding or a FRESH_SNAPSHOT +route that may invoke provider recovery keeps the existing venue quota. This +does not alter freshness, lineage, finality, replay or public entitlement. + ### 15.1 Historical tiers ```text @@ -6160,3 +6177,21 @@ and STOP_LIMIT/TAKE_PROFIT_LIMIT were added to the conditional policy before closure. No provider, runtime role, manifest, data store or order path was mutated. See the main-plan evidence in section 24.2.1 for exact test modules, counts and cleanup. + +### Declared Crypto Release Closure (2026-09-05) + +The grouped route closure in the main plan now has a PASS certificate for the +declared Binance USD-M / OKX Swap consumer scope. See +[`v2.0.12 evidence and notes`](evidence/releases/v2.0.12/RELEASE_NOTES.md). +All299 crypto consumer products are V2_PRIMARY:234 durable and65 on-demand; +four VN products remain explicit V1 exclusions, not implied V2 certification. +The actual full-scope C2 includes300.100s observation, signed reconnect/replay, +allowed V1 fallback-return and blocked-fallback enforcement. Independent +current reads bind to the immutable C2 by digest and satisfy the same source +AND receive-age bounds. Certification preserves typed session evidence for +quiet OBSERVE event feeds; it never treats that evidence as a fresh price. +Closing batches separate hot feed classes from history to avoid head-of-line +age inflation without increasing quotas or weakening deadlines. Public V1/V2 +endpoints, Rust authority and protected execution services are unchanged. +GitHub publication still follows the existing dev -> main release PR and new +tag workflow; certification does not claim that an unpublished tag exists.