diff --git a/docs/calibration-tree-artifacts.md b/docs/calibration-tree-artifacts.md index 49ac11eb..cb8a91d2 100644 --- a/docs/calibration-tree-artifacts.md +++ b/docs/calibration-tree-artifacts.md @@ -370,8 +370,20 @@ Failure meanings and recovery: - `502` from the application means GitHub rejected or did not receive the workflow dispatch; check `GITHUB_ACTIONS_DISPATCH_TOKEN` and its Actions permission. -- A failed GitHub workflow means publication started but did not complete. - Rerun that workflow after correcting the reported error. +- A failed release workflow means publication started but did not complete. + Release reconciliation stops at its first error, so the releases after a + failing one were not reconciled. Rerun that workflow after correcting the + reported error. +- A failed staging workflow either stopped before reconciling any run (for + example, the staging repository was unavailable or listed incompatible + runs), or printed a report in which at least one run is `failed`. In the + second case every run with another outcome was reconciled, and the error + names the failed runs. Every later staging workflow for that country retries + them and fails again until their errors are corrected. Rerunning does not + correct a refused repair of a published tree ("Rebuilt staging index differs + from immutable manifest entry"): the same rebuild differs from the recorded + digest again, so that run keeps failing until the tree's missing files are + restored. - Hugging Face retries non-2xx deliveries and can suspend a repeatedly failing webhook. Correct the failure, re-enable the webhook, and replay the delivery; see the [Hugging Face delivery and retry documentation](https://huggingface.co/docs/hub/webhooks#delivery-and-retries). diff --git a/frontend/packages/calibration-tree/src/publish-calibration-tree-golden.test.ts b/frontend/packages/calibration-tree/src/publish-calibration-tree-golden.test.ts index b1e87ab3..bc35e834 100644 --- a/frontend/packages/calibration-tree/src/publish-calibration-tree-golden.test.ts +++ b/frontend/packages/calibration-tree/src/publish-calibration-tree-golden.test.ts @@ -1,6 +1,7 @@ import { afterEach, expect, spyOn, test } from "bun:test"; import recordedGoldens from "./fixtures/calibration-tree-golden.json"; +import type { CalibrationTreeBundle } from "@policyengine/calibration-tree/calibration-tree-bundle"; import { emptyCalibrationTreeManifest, withCalibrationTreeManifestEntry, @@ -232,6 +233,7 @@ const REFUSED_BUILD_ID = "a".repeat(64); function stagingRepositoryWithRefusedRepair(): { revisionSha: string; store: CalibrationManifestStore; + staleEntry: CalibrationTreeManifestEntry; } { const { revisionSha, files } = ukStagingRepositoryFiles([ REFUSED_STAGING_RUN, @@ -273,7 +275,7 @@ function stagingRepositoryWithRefusedRepair(): { throw new Error("A dry run must not write the manifest."); }, }; - return { revisionSha, store }; + return { revisionSha, store, staleEntry }; } test( @@ -333,3 +335,108 @@ test( }, BUILD_TIMEOUT_MS, ); + +// A store for a reconciliation that writes (not a dry run). It records each +// upload, manifest write, listing and audit in the order the publisher makes +// them. Its manifest starts with the refused run's entry and takes each write +// through the same entry checks as the Blob manifest. A build audits as +// complete once it has been both uploaded and written to the manifest. +function recordingStagingStore(staleEntry: CalibrationTreeManifestEntry) { + const operations: string[] = []; + const uploadedBuildIds = new Set(); + const writtenEntries: CalibrationTreeManifestEntry[] = []; + let storedManifest = withCalibrationTreeManifestEntry( + emptyCalibrationTreeManifest(), + "uk", + staleEntry, + ); + const store: CalibrationManifestStore = { + readManifest: async () => storedManifest, + listBlobs: async () => { + operations.push("list files"); + return new Map(); + }, + audit: async (entry) => { + operations.push(`audit ${entry.stagingRunId}`); + const stored = uploadedBuildIds.has(entry.buildArtifactId) && + writtenEntries.some((written) => written.buildArtifactId === entry.buildArtifactId); + return stored + ? { complete: true, repairable: true, reasons: [], index: null } + : { complete: false, repairable: true, reasons: ["index is missing"], index: null }; + }, + updateManifest: async (entry) => { + operations.push(`write manifest entry for ${entry.stagingRunId}`); + storedManifest = withCalibrationTreeManifestEntry(storedManifest, "uk", entry); + writtenEntries.push(entry); + return storedManifest; + }, + uploadBundle: async (bundle: CalibrationTreeBundle) => { + operations.push(`upload ${bundle.index.build.sourceId}`); + uploadedBuildIds.add(bundle.index.buildArtifactId); + const storedFiles = bundle.files.map((file) => ({ + pathname: file.path, + sha256: file.sha256, + bytes: file.rawBytes, + created: true, + })); + const indexPosition = bundle.files.findIndex((file) => file.part === "index"); + return { files: storedFiles, index: storedFiles[indexPosition] }; + }, + }; + return { + store, + operations, + writtenEntries, + storedManifest: () => storedManifest, + }; +} + +test( + "a reconciliation that writes publishes the run after a refused one and writes nothing for the refused run", + async () => { + const { revisionSha, staleEntry } = stagingRepositoryWithRefusedRepair(); + const recording = recordingStagingStore(staleEntry); + + const report = await withRegisteredRepositories(() => + reconcileStagingBuilds("uk", revisionSha, "unused-blob-token", false, recording.store) + ); + + const publishedOutcome = report.outcomes[1]; + expect(report.outcomes).toEqual([ + { + sourceId: REFUSED_STAGING_RUN.runId, + status: "failed", + buildArtifactId: REFUSED_BUILD_ID, + reason: `Rebuilt staging index differs from immutable manifest entry ${REFUSED_BUILD_ID}.`, + }, + { + sourceId: UNPUBLISHED_STAGING_RUN.runId, + status: "published", + buildArtifactId: expect.stringMatching(/^[0-9a-f]{64}$/), + reason: null, + }, + ]); + // The refused run is audited and then left alone; the next run is + // uploaded, registered, listed again and audited, in that order. + expect(recording.operations).toEqual([ + "list files", + `audit ${REFUSED_STAGING_RUN.runId}`, + `upload ${UNPUBLISHED_STAGING_RUN.runId}`, + `write manifest entry for ${UNPUBLISHED_STAGING_RUN.runId}`, + "list files", + `audit ${UNPUBLISHED_STAGING_RUN.runId}`, + ]); + expect(recording.writtenEntries.map((entry) => [entry.kind, entry.stagingRunId])).toEqual([ + ["staging", UNPUBLISHED_STAGING_RUN.runId], + ]); + // The manifest keeps the refused run's entry exactly as it was and gains + // one entry, for the published run. + const storedBuilds = recording.storedManifest().countries.uk?.builds ?? []; + expect(storedBuilds).toHaveLength(2); + expect(storedBuilds.find((entry) => entry.buildArtifactId === REFUSED_BUILD_ID)) + .toEqual(staleEntry); + expect(storedBuilds.find((entry) => entry.stagingRunId === UNPUBLISHED_STAGING_RUN.runId) + ?.buildArtifactId ?? null).toBe(publishedOutcome.buildArtifactId); + }, + BUILD_TIMEOUT_MS, +);