From c5b7208dce329597151bea6488895ce65e45640e Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:50:59 +0400 Subject: [PATCH 1/2] Test the written path of a failed staging reconciliation #237's tests of the `failed` staging outcome run as dry runs, so none covered a reconciliation that writes. The new test runs one with a recording store. A refused repair of one run is audited and left alone: no upload and no manifest write. The next run is uploaded, written to the manifest, listed again and audited, in that order. The manifest keeps the refused run's entry unchanged and gains exactly one entry. The code is already correct, so the test passes on this branch's base. Changing the per-run error handling to rethrow in a run that writes makes it fail (0 passed, 1 failed). Refs #242 Co-Authored-By: Claude Opus 5.5 --- .../publish-calibration-tree-golden.test.ts | 109 +++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/frontend/packages/calibration-tree/src/publish-calibration-tree-golden.test.ts b/frontend/packages/calibration-tree/src/publish-calibration-tree-golden.test.ts index b1e87ab3..bc35e834 100644 --- a/frontend/packages/calibration-tree/src/publish-calibration-tree-golden.test.ts +++ b/frontend/packages/calibration-tree/src/publish-calibration-tree-golden.test.ts @@ -1,6 +1,7 @@ import { afterEach, expect, spyOn, test } from "bun:test"; import recordedGoldens from "./fixtures/calibration-tree-golden.json"; +import type { CalibrationTreeBundle } from "@policyengine/calibration-tree/calibration-tree-bundle"; import { emptyCalibrationTreeManifest, withCalibrationTreeManifestEntry, @@ -232,6 +233,7 @@ const REFUSED_BUILD_ID = "a".repeat(64); function stagingRepositoryWithRefusedRepair(): { revisionSha: string; store: CalibrationManifestStore; + staleEntry: CalibrationTreeManifestEntry; } { const { revisionSha, files } = ukStagingRepositoryFiles([ REFUSED_STAGING_RUN, @@ -273,7 +275,7 @@ function stagingRepositoryWithRefusedRepair(): { throw new Error("A dry run must not write the manifest."); }, }; - return { revisionSha, store }; + return { revisionSha, store, staleEntry }; } test( @@ -333,3 +335,108 @@ test( }, BUILD_TIMEOUT_MS, ); + +// A store for a reconciliation that writes (not a dry run). It records each +// upload, manifest write, listing and audit in the order the publisher makes +// them. Its manifest starts with the refused run's entry and takes each write +// through the same entry checks as the Blob manifest. A build audits as +// complete once it has been both uploaded and written to the manifest. +function recordingStagingStore(staleEntry: CalibrationTreeManifestEntry) { + const operations: string[] = []; + const uploadedBuildIds = new Set(); + const writtenEntries: CalibrationTreeManifestEntry[] = []; + let storedManifest = withCalibrationTreeManifestEntry( + emptyCalibrationTreeManifest(), + "uk", + staleEntry, + ); + const store: CalibrationManifestStore = { + readManifest: async () => storedManifest, + listBlobs: async () => { + operations.push("list files"); + return new Map(); + }, + audit: async (entry) => { + operations.push(`audit ${entry.stagingRunId}`); + const stored = uploadedBuildIds.has(entry.buildArtifactId) && + writtenEntries.some((written) => written.buildArtifactId === entry.buildArtifactId); + return stored + ? { complete: true, repairable: true, reasons: [], index: null } + : { complete: false, repairable: true, reasons: ["index is missing"], index: null }; + }, + updateManifest: async (entry) => { + operations.push(`write manifest entry for ${entry.stagingRunId}`); + storedManifest = withCalibrationTreeManifestEntry(storedManifest, "uk", entry); + writtenEntries.push(entry); + return storedManifest; + }, + uploadBundle: async (bundle: CalibrationTreeBundle) => { + operations.push(`upload ${bundle.index.build.sourceId}`); + uploadedBuildIds.add(bundle.index.buildArtifactId); + const storedFiles = bundle.files.map((file) => ({ + pathname: file.path, + sha256: file.sha256, + bytes: file.rawBytes, + created: true, + })); + const indexPosition = bundle.files.findIndex((file) => file.part === "index"); + return { files: storedFiles, index: storedFiles[indexPosition] }; + }, + }; + return { + store, + operations, + writtenEntries, + storedManifest: () => storedManifest, + }; +} + +test( + "a reconciliation that writes publishes the run after a refused one and writes nothing for the refused run", + async () => { + const { revisionSha, staleEntry } = stagingRepositoryWithRefusedRepair(); + const recording = recordingStagingStore(staleEntry); + + const report = await withRegisteredRepositories(() => + reconcileStagingBuilds("uk", revisionSha, "unused-blob-token", false, recording.store) + ); + + const publishedOutcome = report.outcomes[1]; + expect(report.outcomes).toEqual([ + { + sourceId: REFUSED_STAGING_RUN.runId, + status: "failed", + buildArtifactId: REFUSED_BUILD_ID, + reason: `Rebuilt staging index differs from immutable manifest entry ${REFUSED_BUILD_ID}.`, + }, + { + sourceId: UNPUBLISHED_STAGING_RUN.runId, + status: "published", + buildArtifactId: expect.stringMatching(/^[0-9a-f]{64}$/), + reason: null, + }, + ]); + // The refused run is audited and then left alone; the next run is + // uploaded, registered, listed again and audited, in that order. + expect(recording.operations).toEqual([ + "list files", + `audit ${REFUSED_STAGING_RUN.runId}`, + `upload ${UNPUBLISHED_STAGING_RUN.runId}`, + `write manifest entry for ${UNPUBLISHED_STAGING_RUN.runId}`, + "list files", + `audit ${UNPUBLISHED_STAGING_RUN.runId}`, + ]); + expect(recording.writtenEntries.map((entry) => [entry.kind, entry.stagingRunId])).toEqual([ + ["staging", UNPUBLISHED_STAGING_RUN.runId], + ]); + // The manifest keeps the refused run's entry exactly as it was and gains + // one entry, for the published run. + const storedBuilds = recording.storedManifest().countries.uk?.builds ?? []; + expect(storedBuilds).toHaveLength(2); + expect(storedBuilds.find((entry) => entry.buildArtifactId === REFUSED_BUILD_ID)) + .toEqual(staleEntry); + expect(storedBuilds.find((entry) => entry.stagingRunId === UNPUBLISHED_STAGING_RUN.runId) + ?.buildArtifactId ?? null).toBe(publishedOutcome.buildArtifactId); + }, + BUILD_TIMEOUT_MS, +); From 0428dc0ae9df1bda40680075fe106c7565407bce Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:51:34 +0400 Subject: [PATCH 2/2] Describe what a failed staging publisher workflow means since #237 "Failure meanings and recovery" said a failed publisher workflow means publication didn't complete. Since #237 that holds for releases only. A failed staging workflow either stopped before any run, or printed a report in which every run without a `failed` outcome was reconciled. Every later staging workflow for that country retries the failed runs. Rerunning doesn't correct a refused repair of a published tree. Documentation only, so there is no red step. Refs #242 Co-Authored-By: Claude Opus 5.5 --- docs/calibration-tree-artifacts.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/calibration-tree-artifacts.md b/docs/calibration-tree-artifacts.md index 49ac11eb..cb8a91d2 100644 --- a/docs/calibration-tree-artifacts.md +++ b/docs/calibration-tree-artifacts.md @@ -370,8 +370,20 @@ Failure meanings and recovery: - `502` from the application means GitHub rejected or did not receive the workflow dispatch; check `GITHUB_ACTIONS_DISPATCH_TOKEN` and its Actions permission. -- A failed GitHub workflow means publication started but did not complete. - Rerun that workflow after correcting the reported error. +- A failed release workflow means publication started but did not complete. + Release reconciliation stops at its first error, so the releases after a + failing one were not reconciled. Rerun that workflow after correcting the + reported error. +- A failed staging workflow either stopped before reconciling any run (for + example, the staging repository was unavailable or listed incompatible + runs), or printed a report in which at least one run is `failed`. In the + second case every run with another outcome was reconciled, and the error + names the failed runs. Every later staging workflow for that country retries + them and fails again until their errors are corrected. Rerunning does not + correct a refused repair of a published tree ("Rebuilt staging index differs + from immutable manifest entry"): the same rebuild differs from the recorded + digest again, so that run keeps failing until the tree's missing files are + restored. - Hugging Face retries non-2xx deliveries and can suspend a repeatedly failing webhook. Correct the failure, re-enable the webhook, and replay the delivery; see the [Hugging Face delivery and retry documentation](https://huggingface.co/docs/hub/webhooks#delivery-and-retries).