diff --git a/README.md b/README.md index c6fe1fe0..e6281e00 100644 --- a/README.md +++ b/README.md @@ -80,9 +80,40 @@ Published-release endpoints accept `country=us|uk|be` (default `us`). | `GET /api/microcosm/staging/runs` | List staging build runs | | `GET /api/microcosm/staging/run?id=` | One staging run's progress and its candidate diagnostics: an uploaded `calibration_diagnostics` telemetry artifact, or the `calibration_diagnostics.json` of the dataset bundle the run staged under `staged//` in the country's release repository (read at the commit the run recorded, digest-verified) | | `GET /api/microcosm/staging/target-diagnostics?id=&...` | Faceted diagnostics for a staging candidate once diagnostics exist | -| `GET /api/microcosm/staging/compare?run=&release=latest` | Diff staging candidate against a published release | +| `GET /api/microcosm/staging/compare?run=&release=latest` | Diff a staging candidate against a published release: by default the current release, or the newest published local-area build for a local-area candidate (see below) | | `GET /api/microcosm/staging/target-change-tree?run=&release=&mode=reported\|shared&...` | One hierarchy level of weighted target-error changes for a staging candidate and an explicit current release | +`/api/microcosm/staging/compare` resolves `release=latest`, its default, for +each candidate: + +- **Local-area candidates.** A run is a local-area candidate when the + `pipeline.id` its `run_manifest.json` records is one of the country's + `staging.local_area_pipelines` in the country registry. The UK lists + `uk-local-candidate`; no other country lists any, so their run manifests are + not read for this. A local-area candidate is compared with the most recently + published local-area build: among release builds in the tree manifest whose + release manifest's `dataset_role` names a local area (such as + `non_default_local_area`), the one with the latest recorded `createdAt`, else + `updatedAt`. The local-area release has no pointer file and is never + promoted, so it has no current build. +- **Every other candidate,** and a local-area candidate when the tree manifest + lists no local-area release build, is compared with the country's current + release. That includes a local-area release published on Hugging Face whose + calibration tree the dashboard has not built yet. + +Only a local-area candidate's response adds `comparison_release`, which names +the release it was compared with: + +| Field | Meaning | +|---|---| +| `release_id` | The release compared with, the same as `a.release_id` | +| `basis` | `newest_local_area_release`, or `current_release` when the tree manifest lists no local-area release build | +| `local_area_candidate` | Always `true` when the field is present | + +An explicit `release=` is compared as requested and adds no +`comparison_release`. Responses for US, Belgian and UK national candidates are +unchanged. + The staging target-change route supports two comparison modes. `reported` uses each release's actual target weights and complete target surface, including added and removed targets. `shared` restricts the calculation to shared targets, @@ -203,6 +234,15 @@ backward-compatible `POPULACE_STAGING_HF_REPO` and with `POPULACE_UK_STAGING_HF_REPO` or `POPULACE_UK_STAGING_HF_REVISION`. Never use a `NEXT_PUBLIC_` variable for a private-repository credential. +A staging registration may also list `local_area_pipelines`: the +`pipeline.id` values, as run manifests record them, of pipelines whose +candidates are local-area builds. The Staging page and +`/api/microcosm/staging/compare` compare those candidates with the newest +published local-area build instead of the current release, and the page names +that release in its validation table and calibration map. The UK lists +`uk-local-candidate`, Microcosm's local-area candidate pipeline; its national +attempts run `uk-frs-calibration`. A country that lists none is unaffected. + The staging contract fixtures under `frontend/packages/calibration-tree/src/fixtures/staging-contract/` are byte-identical copies of Microcosm's canonical producer fixtures. The consumer tests pin each diff --git a/frontend/app/api/microcosm/staging/compare/route.test.ts b/frontend/app/api/microcosm/staging/compare/route.test.ts new file mode 100644 index 00000000..0f9802c7 --- /dev/null +++ b/frontend/app/api/microcosm/staging/compare/route.test.ts @@ -0,0 +1,87 @@ +import { expect, test } from "bun:test"; + +import type { + StagingComparison, + StagingComparisonRelease, +} from "@policyengine/calibration-tree/staging-artifact"; +import { createStagingCompareHandler } from "./route"; + +const DENSE_NEWEST = "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe"; +const LOCAL_CANDIDATE_RUN = "uk-local-candidate-f100-s42-20261019T000432Z-ee5c2dbe"; + +// The route passes a loaded comparison through unchanged, so the stub +// carries only the field the Staging page reads to name the compared +// release, not a whole comparison. +function loadedComparison(runId: string, releaseId: string): StagingComparison { + return { + available: true, + run_id: runId, + a: { release_id: releaseId }, + } as unknown as StagingComparison; +} + +function handler(resolved: StagingComparisonRelease) { + const calls: { resolved: string[]; loaded: Array<[string, string, string]> } = { + resolved: [], + loaded: [], + }; + const get = createStagingCompareHandler({ + resolveComparisonRelease: async (runId, country) => { + calls.resolved.push(`${country}:${runId}`); + return resolved; + }, + loadComparison: async (runId, releaseId, _revalidate, country) => { + calls.loaded.push([country, runId, releaseId]); + return loadedComparison(runId, releaseId); + }, + }); + return { get, calls }; +} + +test("a US candidate's comparison response is unchanged", async () => { + const { get, calls } = handler({ + release_id: "populace-us-2024-spm-receipts-20260923", + basis: "current_release", + local_area_candidate: false, + }); + const response = await get(new Request("http://dashboard.test/api/microcosm/staging/compare?country=us&run=us-run")); + expect(await response.json()).toEqual({ + available: true, + run_id: "us-run", + a: { release_id: "populace-us-2024-spm-receipts-20260923" }, + }); + expect(calls.loaded).toEqual([["us", "us-run", "populace-us-2024-spm-receipts-20260923"]]); +}); + +test("a UK local-area candidate's comparison names the local-area release it uses", async () => { + const resolved: StagingComparisonRelease = { + release_id: DENSE_NEWEST, + basis: "newest_local_area_release", + local_area_candidate: true, + }; + const { get, calls } = handler(resolved); + const response = await get(new Request( + `http://dashboard.test/api/microcosm/staging/compare?country=uk&run=${LOCAL_CANDIDATE_RUN}&release=latest`, + )); + expect(await response.json()).toEqual({ + available: true, + run_id: LOCAL_CANDIDATE_RUN, + a: { release_id: DENSE_NEWEST }, + comparison_release: resolved, + }); + expect(calls.resolved).toEqual([`uk:${LOCAL_CANDIDATE_RUN}`]); + expect(calls.loaded).toEqual([["uk", LOCAL_CANDIDATE_RUN, DENSE_NEWEST]]); +}); + +test("an explicitly requested release is compared as requested", async () => { + const { get, calls } = handler({ + release_id: DENSE_NEWEST, + basis: "newest_local_area_release", + local_area_candidate: true, + }); + await get(new Request( + `http://dashboard.test/api/microcosm/staging/compare?country=uk&run=${LOCAL_CANDIDATE_RUN}&release=some-release`, + )); + expect(calls.resolved).toEqual([]); + expect(calls.loaded).toEqual([["uk", LOCAL_CANDIDATE_RUN, "some-release"]]); +}); diff --git a/frontend/app/api/microcosm/staging/compare/route.ts b/frontend/app/api/microcosm/staging/compare/route.ts index f5ba57a3..15f5015c 100644 --- a/frontend/app/api/microcosm/staging/compare/route.ts +++ b/frontend/app/api/microcosm/staging/compare/route.ts @@ -1,38 +1,75 @@ import { NextResponse } from "next/server"; -import { hasCapability } from "@policyengine/calibration-tree/countries"; +import { hasCapability, type MicrocosmCountry } from "@policyengine/calibration-tree/countries"; import { - loadPointerReleaseId, parseCountry, scrub, } from "@policyengine/calibration-tree/latest-artifact"; -import { loadStagingComparison } from "@policyengine/calibration-tree/staging-artifact"; +import { + loadStagingComparison, + resolveStagingComparisonRelease, + type StagingComparison, + type StagingComparisonRelease, +} from "@policyengine/calibration-tree/staging-artifact"; export const dynamic = "force-dynamic"; export const revalidate = 0; export const runtime = "nodejs"; export const maxDuration = 300; -export async function GET(request: Request) { - const url = new URL(request.url); - const country = parseCountry(url.searchParams.get("country")); - const runId = url.searchParams.get("run")?.trim(); - let release = url.searchParams.get("release")?.trim() || "latest"; - if (!runId && hasCapability(country, "staging")) { - return NextResponse.json({ detail: "Provide a staging run id via ?run=." }, { status: 400 }); - } - try { - if (release === "latest" && hasCapability(country, "staging")) { - release = (await loadPointerReleaseId(300, country)).release_id; +export interface StagingCompareRouteDependencies { + resolveComparisonRelease( + runId: string, + country: MicrocosmCountry, + ): Promise; + loadComparison( + runId: string, + releaseId: string, + revalidate: number, + country: MicrocosmCountry, + ): Promise; +} + +export function createStagingCompareHandler( + dependencies: StagingCompareRouteDependencies = { + resolveComparisonRelease: (runId, country) => + resolveStagingComparisonRelease(runId, country), + loadComparison: loadStagingComparison, + }, +) { + return async function GET(request: Request) { + const url = new URL(request.url); + const country = parseCountry(url.searchParams.get("country")); + const runId = url.searchParams.get("run")?.trim(); + let release = url.searchParams.get("release")?.trim() || "latest"; + if (!runId && hasCapability(country, "staging")) { + return NextResponse.json({ detail: "Provide a staging run id via ?run=." }, { status: 400 }); } - return NextResponse.json( - scrub(await loadStagingComparison(runId ?? "", release, revalidate, country)), - { headers: { "Cache-Control": "no-store" } }, - ); - } catch (error) { - return NextResponse.json( - { detail: error instanceof Error ? error.message : String(error) }, - { status: 502 }, - ); - } + try { + // "latest" is the current release for a national candidate and the + // newest published local-area release for a local-area candidate; the + // response names the local-area choice so the page can say so. + let comparisonRelease: StagingComparisonRelease | null = null; + if (release === "latest" && hasCapability(country, "staging")) { + comparisonRelease = await dependencies.resolveComparisonRelease(runId ?? "", country); + release = comparisonRelease.release_id; + } + const comparison = await dependencies.loadComparison(runId ?? "", release, revalidate, country); + return NextResponse.json( + scrub( + comparisonRelease?.local_area_candidate + ? { ...comparison, comparison_release: comparisonRelease } + : comparison, + ), + { headers: { "Cache-Control": "no-store" } }, + ); + } catch (error) { + return NextResponse.json( + { detail: error instanceof Error ? error.message : String(error) }, + { status: 502 }, + ); + } + }; } + +export const GET = createStagingCompareHandler(); diff --git a/frontend/components/microcosm/calibration-explorer-map.tsx b/frontend/components/microcosm/calibration-explorer-map.tsx index 485251f2..812bff40 100644 --- a/frontend/components/microcosm/calibration-explorer-map.tsx +++ b/frontend/components/microcosm/calibration-explorer-map.tsx @@ -406,6 +406,8 @@ interface CalibrationExplorerMapProps { stagingRunId?: string; stagingComparison?: { releaseId: string; + /** Names that release, e.g. "Local-area release"; defaults to "Current release". */ + releaseLabel?: string; rows: MicrocosmComparisonRow[]; weightedTargetErrorChange: number | null; }; @@ -506,6 +508,7 @@ function CalibrationExplorerMapForSource({ selectedTargetChange, stagingComparison.weightedTargetErrorChange, selectedTarget, + stagingComparison.releaseLabel, ) : null; diff --git a/frontend/components/microcosm/microcosm-staging-view.tsx b/frontend/components/microcosm/microcosm-staging-view.tsx index 17879880..0a27d54d 100644 --- a/frontend/components/microcosm/microcosm-staging-view.tsx +++ b/frontend/components/microcosm/microcosm-staging-view.tsx @@ -45,6 +45,7 @@ import { formatStagingStatus, } from "@/lib/microcosm/staging-status"; import { formatWeightedTargetError } from "@/lib/microcosm/target-change-visualization"; +import { stagingComparisonBaseline } from "@/lib/microcosm/calibration-selection"; type LossKind = "normalized_target_loss" | "raw_optimizer_objective" | undefined; @@ -694,7 +695,7 @@ const VALIDATION_METHOD_HELP = { reformWithin10: "Measures the share of scored out-of-sample reforms whose estimated effect is within 10% of the external benchmark. Out-of-sample means that the reform includes values that are not used as calibration targets; higher is better.", targetCoverage: - "Counts the complete set of calibration targets available in each release, including targets that are not shared. The verdict reports how many targets the candidate adds and removes relative to the current release.", + "Counts the complete set of calibration targets available in each release, including targets that are not shared. The verdict reports how many targets the candidate adds and removes relative to the release it is compared with.", } as const; function sideStats(errors: number[]): SideStats { @@ -845,6 +846,9 @@ function MicrocosmStagingRunsView({ runData?.has_calibration ? selectedRun : undefined, "latest", ); + // A local-area candidate is compared with the local-area release, not the + // current (national) release; the page names which. + const comparisonBaseline = stagingComparisonBaseline(compareData?.comparison_release); // Fit stats on the targets both sides share — the honest better-or-worse basis. const commonStats = useMemo(() => { const rows = compareData?.rows ?? []; @@ -1029,7 +1033,7 @@ function MicrocosmStagingRunsView({ title="Candidate validation" > @@ -1038,6 +1042,7 @@ function MicrocosmStagingRunsView({ {hasCandidateValidation && (
@@ -1045,7 +1050,7 @@ function MicrocosmStagingRunsView({ Validation point - Current release + {comparisonBaseline.releaseLabel} Candidate Verdict About @@ -1183,6 +1188,7 @@ function MicrocosmStagingRunsView({ comparisonReleaseId={ compareData?.summary ? compareData.a.release_id : undefined } + comparisonRelease={compareData?.comparison_release} comparisonRows={compareData?.rows} weightedTargetErrorChange={weightedTargetErrorChange} pageIntroHeight={pageIntroHeight} diff --git a/frontend/components/microcosm/staging-calibration-map-panel.tsx b/frontend/components/microcosm/staging-calibration-map-panel.tsx index cfaa2815..9e9398d9 100644 --- a/frontend/components/microcosm/staging-calibration-map-panel.tsx +++ b/frontend/components/microcosm/staging-calibration-map-panel.tsx @@ -1,6 +1,6 @@ "use client"; -import { useEffect, useMemo, useState } from "react"; +import { useMemo, useState } from "react"; import { CalibrationExplorerMap } from "@/components/microcosm/calibration-explorer-map"; import { StagingTargetChangeMap } from "@/components/microcosm/staging-target-change-map"; @@ -12,10 +12,14 @@ import { type MicrocosmCalibrationBuild, type MicrocosmComparisonRow, } from "@/lib/api/hooks/use-microcosm"; +import { calibrationBuildsNewestFirst } from "@policyengine/calibration-tree/calibration-tree-manifest"; import { calibrationBuildManifestState, - calibrationBuildsNewestFirst, + comparisonMapBuilds, + stagingComparisonBaseline, + type ComparisonMapBuildPicks, } from "@/lib/microcosm/calibration-selection"; +import type { StagingComparisonRelease } from "@policyengine/calibration-tree/staging-artifact"; import { targetChangeMapIdentity } from "@/lib/microcosm/target-change-visualization"; type CalibrationMapView = "candidate" | "comparison"; @@ -61,14 +65,16 @@ function BuildSelect({ function CalibrationMapViewSelect({ value, + comparisonLabel, onChange, }: { value: CalibrationMapView; + comparisonLabel: string; onChange: (value: CalibrationMapView) => void; }) { const options: Array<{ value: CalibrationMapView; label: string }> = [ { value: "candidate", label: "Candidate fit" }, - { value: "comparison", label: "Change from current release" }, + { value: "comparison", label: comparisonLabel }, ]; return ( @@ -103,6 +109,7 @@ function CalibrationMapViewSelect({ export function StagingCalibrationMapPanel({ runId, comparisonReleaseId, + comparisonRelease, comparisonRows, weightedTargetErrorChange, pageIntroHeight, @@ -112,6 +119,7 @@ export function StagingCalibrationMapPanel({ }: { runId: string; comparisonReleaseId?: string; + comparisonRelease?: StagingComparisonRelease; comparisonRows?: MicrocosmComparisonRow[]; weightedTargetErrorChange: number | null; pageIntroHeight: number; @@ -120,6 +128,7 @@ export function StagingCalibrationMapPanel({ comparisonDetail?: string | null; }) { const [view, setView] = useState("candidate"); + const comparisonBaseline = stagingComparisonBaseline(comparisonRelease); const { data: buildManifest, isLoading: buildManifestLoading, @@ -139,23 +148,21 @@ export function StagingCalibrationMapPanel({ const selectedRunBuild = builds.find( (build) => build.stagingRunId === runId, ); - const [currentBuildArtifactId, setCurrentBuildArtifactId] = useState(""); - const [candidateBuildArtifactId, setCandidateBuildArtifactId] = useState(""); - - useEffect(() => { - setCurrentBuildArtifactId((current) => - builds.some((build) => build.buildArtifactId === current) - ? current - : buildManifest?.latestReleaseBuildArtifactId ?? - builds[0]?.buildArtifactId ?? - "", - ); - setCandidateBuildArtifactId((candidate) => - candidate && builds.some((build) => build.buildArtifactId === candidate) - ? candidate - : selectedRunBuild?.buildArtifactId ?? "", - ); - }, [buildManifest?.latestReleaseBuildArtifactId, builds, selectedRunBuild?.buildArtifactId]); + // Only the user's picks are state; the selectors' defaults follow the + // build manifest and the comparison as each responds. + const [buildPicks, setBuildPicks] = useState({ + from: null, + to: null, + }); + const { + from: currentBuildArtifactId, + to: candidateBuildArtifactId, + } = comparisonMapBuilds(buildPicks, { + builds, + latestReleaseBuildArtifactId: buildManifest?.latestReleaseBuildArtifactId, + comparisonReleaseId, + candidateBuildArtifactId: selectedRunBuild?.buildArtifactId, + }); const immutableComparison = Boolean( currentBuildArtifactId && candidateBuildArtifactId, @@ -163,7 +170,14 @@ export function StagingCalibrationMapPanel({ return ( } + description={view === "comparison" ? comparisonBaseline.note ?? undefined : undefined} + actions={ + + } > {view === "candidate" ? ( + setBuildPicks((picks) => ({ ...picks, from: buildArtifactId })) + } /> + setBuildPicks((picks) => ({ ...picks, to: buildArtifactId })) + } />
) : null} @@ -249,6 +268,7 @@ export function StagingCalibrationMapPanel({ weightedTargetErrorChange={ immutableComparison ? null : weightedTargetErrorChange } + comparisonReleaseLabel={comparisonBaseline.releaseLabel} /> ) : comparisonLoading ? ( = { - reported: - "Uses each release's actual target weights and complete target surface. Added targets contribute candidate error, while removed targets subtract their current-release contribution.", - shared: - "Uses only targets present in both releases and applies the same pooled target weights to both. Each release still uses its own verified benchmark, scale, cap, and capped error.", -}; - interface LaidGroup { group: CalibrationTreemapGroup; rect: Placed; @@ -248,13 +242,17 @@ export function StagingTargetChangeMap({ currentBuildArtifactId, candidateBuildArtifactId, weightedTargetErrorChange, + comparisonReleaseLabel = "Current release", }: { runId?: string; releaseId?: string; currentBuildArtifactId?: string; candidateBuildArtifactId?: string; weightedTargetErrorChange: number | null; + /** Names the release the candidate is compared with, e.g. "Local-area release". */ + comparisonReleaseLabel?: string; }) { + const modeHelp = targetChangeModeHelp(comparisonReleaseLabel); const [state, dispatch] = useReducer(explorerReducer, undefined, createExplorerState); const [mode, setMode] = useState("reported"); const [expanded, setExpanded] = useState<{ @@ -339,6 +337,7 @@ export function StagingTargetChangeMap({ data.selectedTarget, data.summary?.netChange ?? weightedTargetErrorChange, data.selectedTarget.candidateDetail as MicrocosmTargetRow | null, + comparisonReleaseLabel, ) : null; const filtersActive = hasExplorerFilters(state); @@ -355,12 +354,12 @@ export function StagingTargetChangeMap({ { value: "reported", label: "Reported error change", - tooltip: MODE_HELP.reported, + tooltip: modeHelp.reported, }, { value: "shared", label: "Shared-target comparison", - tooltip: MODE_HELP.shared, + tooltip: modeHelp.shared, }, ]} onChange={(nextMode) => { @@ -563,11 +562,7 @@ export function StagingTargetChangeMap({ text: "Candidate dataset decreased absolute error by 5.00 percentage points relative to the current release dataset.", tone: "positive", }); + expect(presentation.eyebrow).toBe("Candidate calibration target"); }); test("omits the unavailable release from added and removed target chart labels", () => { @@ -144,4 +145,69 @@ test("omits the unavailable release from added and removed target chart labels", text: "Current release contains this target, but the candidate does not.", tone: "neutral", }); + expect(removed.eyebrow).toBe("Current release calibration target"); +}); + +const ADDED_TARGET_CHANGE: TargetChangeRow = { + ...TARGET_CHANGE, + comparison_status: "added", + match_kind: null, + current_name: null, + current_representation: null, + current: null, + currentDetail: null, +}; +const REMOVED_TARGET_CHANGE: TargetChangeRow = { + ...TARGET_CHANGE, + comparison_status: "removed", + match_kind: null, + candidate_name: null, + candidate_representation: null, + candidate: null, + candidateDetail: null, +}; + +test("names the release a local-area candidate is compared with", () => { + const shared = stagingTargetDetailPresentation( + TARGET_CHANGE, + -0.07, + TARGET_CHANGE.candidateDetail, + "Local-area release", + ); + expect(shared.metrics.map((metric) => [metric.label, metric.caption])).toEqual([ + ["Official target", undefined], + ["Local-area release estimate", undefined], + ["Candidate estimate", undefined], + ["Local-area release error", "Relative to target"], + ["Candidate release error", "Relative to target"], + ["Weighted target error change", "Candidate dataset minus local-area release"], + ]); + expect(shared.afterCalibrationSeries.map((series) => [series.label, series.shortLabel])).toEqual([ + ["Local-area release after calibration", "Local-area release"], + ["Candidate after calibration", "Candidate"], + ]); + expect(shared.fitSummary?.text).toBe( + "Candidate dataset decreased absolute error by 5.00 percentage points relative to the local-area release dataset.", + ); + + const added = stagingTargetDetailPresentation( + ADDED_TARGET_CHANGE, + -0.07, + TARGET_CHANGE.candidateDetail, + "Local-area release", + ); + expect(added.fitSummary?.text).toBe( + "Candidate contains this target, but the local-area release does not.", + ); + + const removed = stagingTargetDetailPresentation( + REMOVED_TARGET_CHANGE, + -0.07, + null, + "Local-area release", + ); + expect(removed.eyebrow).toBe("Local-area release calibration target"); + expect(removed.fitSummary?.text).toBe( + "Local-area release contains this target, but the candidate does not.", + ); }); diff --git a/frontend/components/microcosm/staging-target-detail-presentation.ts b/frontend/components/microcosm/staging-target-detail-presentation.ts index 27a06a37..2a7461de 100644 --- a/frontend/components/microcosm/staging-target-detail-presentation.ts +++ b/frontend/components/microcosm/staging-target-detail-presentation.ts @@ -7,9 +7,14 @@ import { } from "@/components/microcosm/microcosm-target-detail"; import type { MicrocosmTargetRow } from "@/lib/api/hooks/use-microcosm"; import type { TargetChangeRow } from "@policyengine/calibration-tree/target-change"; -import { formatTargetChange } from "@/lib/microcosm/target-change-visualization"; +import { + formatTargetChange, + sentenceCaseToPhrase, +} from "@/lib/microcosm/target-change-visualization"; export interface StagingTargetDetailPresentation { + /** Which side the shown detail row comes from. */ + eyebrow: string; metrics: MicrocosmTargetDetailMetric[]; afterCalibrationSeries: MicrocosmTargetErrorSeries[]; fitSummary: MicrocosmTargetErrorSummary | null; @@ -55,6 +60,7 @@ function rowFromDetail( function candidateComparisonSummary( currentError: ReturnType, candidateError: ReturnType, + comparisonReleasePhrase: string, ): MicrocosmTargetErrorSummary | null { if ( currentError.kind !== candidateError.kind || @@ -66,7 +72,7 @@ function candidateComparisonSummary( const change = Math.abs(candidateError.value) - Math.abs(currentError.value); if (Math.abs(change) < 0.00005) { return { - text: "Candidate dataset left absolute error unchanged relative to the current release dataset.", + text: `Candidate dataset left absolute error unchanged relative to the ${comparisonReleasePhrase} dataset.`, tone: "neutral", }; } @@ -76,16 +82,23 @@ function candidateComparisonSummary( : `${(Math.abs(change) * 100).toFixed(Math.abs(change) >= 0.1 ? 1 : 2)} percentage points`; const increased = change > 0; return { - text: `Candidate dataset ${increased ? "increased" : "decreased"} absolute error by ${amount} relative to the current release dataset.`, + text: `Candidate dataset ${increased ? "increased" : "decreased"} absolute error by ${amount} relative to the ${comparisonReleasePhrase} dataset.`, tone: increased ? "negative" : "positive", }; } +/** + * The staging target detail's metrics, chart series and summary. + * `comparisonReleaseLabel` names the release the candidate is compared with, + * such as "Local-area release" for a UK local-area candidate. + */ export function stagingTargetDetailPresentation( targetChange: TargetChangeRow | null, weightedTargetErrorChange: number | null, candidateFallback: MicrocosmTargetRow | null, + comparisonReleaseLabel = "Current release", ): StagingTargetDetailPresentation { + const comparisonReleasePhrase = sentenceCaseToPhrase(comparisonReleaseLabel); const currentRow = rowFromDetail(targetChange?.currentDetail ?? null); const candidateRow = rowFromDetail(targetChange?.candidateDetail ?? null) ?? candidateFallback; const currentError = targetError(currentRow); @@ -105,8 +118,8 @@ export function stagingTargetDetailPresentation( const afterCalibrationSeries: MicrocosmTargetErrorSeries[] = []; if (currentRow) { afterCalibrationSeries.push({ - label: "Current release after calibration", - shortLabel: "Current release", + label: `${comparisonReleaseLabel} after calibration`, + shortLabel: comparisonReleaseLabel, value: currentError.value, tone: "primary", }); @@ -121,6 +134,9 @@ export function stagingTargetDetailPresentation( } return { + eyebrow: targetChange?.candidateDetail + ? "Candidate calibration target" + : `${comparisonReleaseLabel} calibration target`, metrics: [ { label: "Official target", @@ -128,7 +144,7 @@ export function stagingTargetDetailPresentation( caption: unit, }, { - label: "Current release estimate", + label: `${comparisonReleaseLabel} estimate`, value: targetEstimateText(currentEstimate), caption: unit, }, @@ -138,7 +154,7 @@ export function stagingTargetDetailPresentation( caption: unit, }, { - label: "Current release error", + label: `${comparisonReleaseLabel} error`, value: targetErrorText(currentError.kind, currentError.value, true), caption: currentError.kind === "absolute" ? "Absolute difference" : "Relative to target", tone: errorTone(currentError.kind, currentError.value), @@ -152,7 +168,7 @@ export function stagingTargetDetailPresentation( { label: "Weighted target error change", value: formatTargetChange(weightedTargetErrorChange), - caption: "Candidate dataset minus current release", + caption: `Candidate dataset minus ${comparisonReleasePhrase}`, tone: weightedTargetErrorChange == null || weightedTargetErrorChange === 0 ? "neutral" @@ -163,15 +179,15 @@ export function stagingTargetDetailPresentation( ], afterCalibrationSeries, fitSummary: currentRow && candidateRow - ? candidateComparisonSummary(currentError, candidateError) + ? candidateComparisonSummary(currentError, candidateError, comparisonReleasePhrase) : candidateRow ? { - text: "Candidate contains this target, but the current release does not.", + text: `Candidate contains this target, but the ${comparisonReleasePhrase} does not.`, tone: "neutral", } : currentRow ? { - text: "Current release contains this target, but the candidate does not.", + text: `${comparisonReleaseLabel} contains this target, but the candidate does not.`, tone: "neutral", } : null, diff --git a/frontend/lib/api/hooks/use-microcosm.ts b/frontend/lib/api/hooks/use-microcosm.ts index 9de1cc6c..f5740cea 100644 --- a/frontend/lib/api/hooks/use-microcosm.ts +++ b/frontend/lib/api/hooks/use-microcosm.ts @@ -16,6 +16,7 @@ import { } from "@/lib/api/cache-policy"; import { withBasePath } from "@/lib/base-path"; import type { BuildRunSource } from "@/lib/microcosm/build-monitor"; +import type { StagingComparisonRelease } from "@policyengine/calibration-tree/staging-artifact"; import type { BuildRunResponse, BuildRunsResponse } from "@/lib/microcosm/build-runs"; import type { ExplorerState } from "@policyengine/calibration-tree/calibration-explorer"; import type { CalibrationTreeResponse } from "@policyengine/calibration-tree/calibration-tree"; @@ -812,7 +813,12 @@ export function useMicrocosmStagingCompare(runId?: string, release = "latest") { return useQuery({ queryKey: ["microcosm", "staging", "compare", country, runId, release], queryFn: () => - apiGet( + apiGet( "/microcosm/staging/compare", { run: runId, release, country }, ), diff --git a/frontend/lib/microcosm/calibration-selection.test.ts b/frontend/lib/microcosm/calibration-selection.test.ts index 34262156..e55d4546 100644 --- a/frontend/lib/microcosm/calibration-selection.test.ts +++ b/frontend/lib/microcosm/calibration-selection.test.ts @@ -1,10 +1,13 @@ import { expect, test } from "bun:test"; +import { microcosmStagingTargetChangeSourceParameters } from "@/lib/api/hooks/use-microcosm"; import { calibrationBuildManifestState, - calibrationBuildsNewestFirst, + comparisonMapBuilds, isStagingSelection, publishedReleaseSelection, + stagingComparisonBaseline, + type ComparisonMapBuildSources, } from "./calibration-selection"; test("recognizes obsolete staging selectors without treating releases as staging", () => { @@ -22,46 +25,6 @@ test("published-release pages discard staging candidate selectors", () => { expect(publishedReleaseSelection(undefined)).toBe(""); }); -test("orders calibration builds by instant across RFC 3339 offsets", () => { - const builds = [ - { - buildArtifactId: "offset-earlier", - createdAt: "2026-01-01T00:30:00+02:00", - updatedAt: "2026-01-01T00:30:00+02:00", - }, - { - buildArtifactId: "utc-later", - createdAt: "2026-01-01T00:00:00Z", - updatedAt: "2026-01-01T00:00:00Z", - }, - { - buildArtifactId: "same-instant-b", - createdAt: null, - updatedAt: "2025-12-31T19:00:00-05:00", - }, - { - buildArtifactId: "same-instant-a", - createdAt: "2026-01-01T00:00:00+00:00", - updatedAt: "2026-01-01T00:00:00+00:00", - }, - ]; - - expect( - calibrationBuildsNewestFirst(builds).map((build) => build.buildArtifactId), - ).toEqual([ - "same-instant-a", - "same-instant-b", - "utc-later", - "offset-earlier", - ]); - expect(builds.map((build) => build.buildArtifactId)).toEqual([ - "offset-earlier", - "utc-later", - "same-instant-b", - "same-instant-a", - ]); -}); - test("distinguishes build-manifest loading, error, empty, and ready states", () => { expect(calibrationBuildManifestState({ hasData: false, @@ -95,17 +58,133 @@ test("distinguishes build-manifest loading, error, empty, and ready states", () })).toBe("ready"); }); -test("an undated calibration build sorts after every dated one", () => { - const builds = [ - { buildArtifactId: "undated-b", createdAt: null, updatedAt: null }, - { - buildArtifactId: "dated", - createdAt: "2026-10-04T18:57:22Z", - updatedAt: "2026-10-04T18:57:22Z", - }, - { buildArtifactId: "undated-a", createdAt: null, updatedAt: null }, - ]; - expect( - calibrationBuildsNewestFirst(builds).map((build) => build.buildArtifactId), - ).toEqual(["dated", "undated-a", "undated-b"]); +const DENSE_NEWEST = "microcosm-uk-2024-25-dense-20261019T000432Z-ee5c2dbe"; +const NATIONAL_CURRENT = "microcosm-uk-2024-25-national-20261019T083323Z-6cba5d71"; + +const PUBLISHED_BUILDS = [ + { buildArtifactId: "national-build", kind: "release" as const, releaseId: NATIONAL_CURRENT }, + { buildArtifactId: "dense-build", kind: "release" as const, releaseId: DENSE_NEWEST }, + { buildArtifactId: "candidate-build", kind: "staging" as const, releaseId: null }, +]; +const NO_PICKS = { from: null, to: null }; +const NOTHING_LOADED: ComparisonMapBuildSources = { + builds: [], + latestReleaseBuildArtifactId: undefined, + comparisonReleaseId: undefined, + candidateBuildArtifactId: undefined, +}; +// A local-area candidate the publisher has not built a tree for yet. +const MANIFEST_LOADED: ComparisonMapBuildSources = { + builds: PUBLISHED_BUILDS, + latestReleaseBuildArtifactId: "national-build", + comparisonReleaseId: undefined, + candidateBuildArtifactId: undefined, +}; +const COMPARISON_LOADED: ComparisonMapBuildSources = { + ...MANIFEST_LOADED, + comparisonReleaseId: DENSE_NEWEST, +}; + +test("the comparison map's From build follows the comparison when it arrives after the build manifest", () => { + // Each render's selector values, in the order the page usually loads. + expect(comparisonMapBuilds(NO_PICKS, NOTHING_LOADED)).toEqual({ from: "", to: "" }); + // The build manifest responds first: the manifest's latest release. + expect(comparisonMapBuilds(NO_PICKS, MANIFEST_LOADED)).toEqual({ + from: "national-build", + to: "", + }); + // `/microcosm/staging/compare` then names the local-area release. + const shown = comparisonMapBuilds(NO_PICKS, COMPARISON_LOADED); + expect(shown).toEqual({ from: "dense-build", to: "" }); + // So the "Change from local-area release" tab's live comparison reads the + // local-area build, not the national one. + expect(microcosmStagingTargetChangeSourceParameters(DENSE_NEWEST, shown.from)).toEqual({ + build: "dense-build", + }); +}); + +test("the comparison map's From build is the same when the comparison arrives first", () => { + expect(comparisonMapBuilds(NO_PICKS, { ...NOTHING_LOADED, comparisonReleaseId: DENSE_NEWEST })) + .toEqual({ from: "", to: "" }); + expect(comparisonMapBuilds(NO_PICKS, COMPARISON_LOADED).from).toBe("dense-build"); +}); + +test("a build the user picks holds while the comparison loads or changes", () => { + const picked = { from: "national-build", to: null }; + expect(comparisonMapBuilds(picked, MANIFEST_LOADED).from).toBe("national-build"); + expect(comparisonMapBuilds(picked, COMPARISON_LOADED).from).toBe("national-build"); + // A pick that is no longer a published build gives way to the default. + expect(comparisonMapBuilds({ from: "withdrawn-build", to: null }, COMPARISON_LOADED).from) + .toBe("dense-build"); +}); + +test("the To selector starts at the run's published build and keeps a pick of the live candidate", () => { + const withCandidateBuild = { ...COMPARISON_LOADED, candidateBuildArtifactId: "candidate-build" }; + expect(comparisonMapBuilds(NO_PICKS, withCandidateBuild).to).toBe("candidate-build"); + // The empty value is the selector's live-candidate option. + expect(comparisonMapBuilds({ from: null, to: "" }, withCandidateBuild).to).toBe(""); + expect(comparisonMapBuilds({ from: null, to: "national-build" }, withCandidateBuild).to) + .toBe("national-build"); + expect(comparisonMapBuilds({ from: null, to: "withdrawn-build" }, withCandidateBuild).to) + .toBe("candidate-build"); +}); + +test("without a published build of the comparison's release, From starts at the manifest's latest release", () => { + // US, Belgium and UK national candidates before the comparison responds. + expect(comparisonMapBuilds(NO_PICKS, MANIFEST_LOADED).from).toBe("national-build"); + expect(comparisonMapBuilds(NO_PICKS, { + ...MANIFEST_LOADED, + comparisonReleaseId: "unpublished-release", + }).from).toBe("national-build"); + expect(comparisonMapBuilds(NO_PICKS, { + ...MANIFEST_LOADED, + latestReleaseBuildArtifactId: null, + }).from).toBe("national-build"); +}); + +test("the Staging page names the release a candidate is compared with", () => { + const current = { + releaseLabel: "Current release", + tabLabel: "Change from current release", + note: null, + }; + expect(stagingComparisonBaseline(undefined)).toEqual(current); + expect(stagingComparisonBaseline({ + release_id: NATIONAL_CURRENT, + basis: "current_release", + local_area_candidate: false, + })).toEqual(current); + expect(stagingComparisonBaseline({ + release_id: DENSE_NEWEST, + basis: "newest_local_area_release", + local_area_candidate: true, + })).toEqual({ + releaseLabel: "Local-area release", + tabLabel: "Change from local-area release", + note: + `This local-area candidate is compared with ${DENSE_NEWEST}, the most recently ` + + "published build of the local-area release. That release is never promoted, " + + "so it has no current build.", + }); +}); + +test("the fallback note says the dashboard has no local-area build, not that none was published", () => { + // The fallback is chosen from the dashboard's build manifest alone. A + // local-area release published on Hugging Face whose calibration tree the + // dashboard has not built yet (tree publishing paused, or its role + // unreadable) also lands here, so the note must not say that no local-area + // release has been published. + const fallback = stagingComparisonBaseline({ + release_id: NATIONAL_CURRENT, + basis: "current_release", + local_area_candidate: true, + }); + expect(fallback).toEqual({ + releaseLabel: "Current release", + tabLabel: "Change from current release", + note: + "The dashboard has no published calibration build of a local-area release, so " + + `this local-area candidate is compared with the current release, ${NATIONAL_CURRENT}, ` + + "which calibrates a different set of targets.", + }); }); diff --git a/frontend/lib/microcosm/calibration-selection.ts b/frontend/lib/microcosm/calibration-selection.ts index d1c79d0b..d86a70eb 100644 --- a/frontend/lib/microcosm/calibration-selection.ts +++ b/frontend/lib/microcosm/calibration-selection.ts @@ -1,3 +1,5 @@ +import type { StagingComparisonRelease } from "@policyengine/calibration-tree/staging-artifact"; + // Older dashboard links used `staging:` on published-release pages. // Those pages now discard the obsolete selector; staging candidates are // reviewed only on the Staging candidates page. @@ -14,33 +16,6 @@ export function publishedReleaseSelection( return isStagingSelection(selection) ? "" : (selection ?? ""); } -interface TimestampedCalibrationBuild { - buildArtifactId: string; - createdAt: string | null; - updatedAt: string | null; -} - -function buildInstant(build: TimestampedCalibrationBuild): number { - const value = build.createdAt ?? build.updatedAt; - const time = value == null ? Number.NaN : Date.parse(value); - // An undated build sorts last rather than wherever NaN lands. - return Number.isFinite(time) ? time : Number.NEGATIVE_INFINITY; -} - -/** Order build records by their actual RFC 3339 instant, not timestamp text. */ -export function calibrationBuildsNewestFirst< - T extends TimestampedCalibrationBuild, ->(builds: readonly T[]): T[] { - return [...builds].sort((left, right) => { - const leftTime = buildInstant(left); - const rightTime = buildInstant(right); - return ( - rightTime - leftTime || - left.buildArtifactId.localeCompare(right.buildArtifactId) - ); - }); -} - export type CalibrationBuildManifestState = | "loading" | "error" @@ -57,3 +32,98 @@ export function calibrationBuildManifestState(options: { if (options.isLoading) return "loading"; return options.hasError ? "error" : "loading"; } + +interface ComparableCalibrationBuild { + buildArtifactId: string; + kind: "release" | "staging"; + releaseId: string | null; +} + +/** The builds the user picked in the comparison map; null until they pick. */ +export interface ComparisonMapBuildPicks { + from: string | null; + /** The empty string is the "To" selector's live-candidate option. */ + to: string | null; +} + +export interface ComparisonMapBuildSources { + /** Published builds, newest first. */ + builds: readonly ComparableCalibrationBuild[]; + latestReleaseBuildArtifactId: string | null | undefined; + /** The release `/microcosm/staging/compare` names; undefined until it responds. */ + comparisonReleaseId: string | undefined; + /** The selected staging run's published build, when it has one. */ + candidateBuildArtifactId: string | undefined; +} + +/** + * The builds the Staging comparison map's "From" and "To" selectors show. + * They are worked out on every render from the user's picks, never stored + * as defaults, because the build manifest usually responds before + * `/microcosm/staging/compare`: "From" starts at the manifest's latest + * release and moves to the build of the release the candidate is compared + * with once the comparison names it. A pick that is not a published build is + * ignored. Without picks: + * + * - "From" is the published build of the comparison's release, otherwise the + * manifest's latest release, otherwise the newest build; + * - "To" is the selected run's published build, otherwise the live candidate. + */ +export function comparisonMapBuilds( + picks: ComparisonMapBuildPicks, + sources: ComparisonMapBuildSources, +): { from: string; to: string } { + const isPublished = (buildArtifactId: string | null): buildArtifactId is string => + buildArtifactId != null && + sources.builds.some((build) => build.buildArtifactId === buildArtifactId); + const comparisonBuild = sources.comparisonReleaseId + ? sources.builds.find( + (build) => build.kind === "release" && build.releaseId === sources.comparisonReleaseId, + ) + : undefined; + const defaultFrom = comparisonBuild?.buildArtifactId ?? + sources.latestReleaseBuildArtifactId ?? + sources.builds[0]?.buildArtifactId ?? + ""; + return { + from: isPublished(picks.from) ? picks.from : defaultFrom, + to: picks.to === "" || isPublished(picks.to) + ? picks.to + : sources.candidateBuildArtifactId ?? "", + }; +} + +export interface StagingComparisonBaseline { + /** Names the compared release: the table column and the map's labels. */ + releaseLabel: string; + tabLabel: string; + note: string | null; +} + +/** How the Staging page names the release a candidate is compared with. */ +export function stagingComparisonBaseline( + comparisonRelease: StagingComparisonRelease | undefined, +): StagingComparisonBaseline { + if (comparisonRelease?.basis === "newest_local_area_release") { + return { + releaseLabel: "Local-area release", + tabLabel: "Change from local-area release", + note: + `This local-area candidate is compared with ${comparisonRelease.release_id}, the most recently ` + + "published build of the local-area release. That release is never promoted, " + + "so it has no current build.", + }; + } + // The fallback is decided from the dashboard's build manifest alone: a + // local-area release published on Hugging Face whose calibration tree the + // dashboard has not built yet also lands here. + return { + releaseLabel: "Current release", + tabLabel: "Change from current release", + note: comparisonRelease?.local_area_candidate + ? "The dashboard has no published calibration build of a local-area release, so " + + `this local-area candidate is compared with the current release, ${comparisonRelease.release_id}, ` + + "which calibrates a different set of targets." + : null, + }; +} diff --git a/frontend/lib/microcosm/target-change-visualization.test.ts b/frontend/lib/microcosm/target-change-visualization.test.ts index e8bb44ba..6837fbaa 100644 --- a/frontend/lib/microcosm/target-change-visualization.test.ts +++ b/frontend/lib/microcosm/target-change-visualization.test.ts @@ -12,6 +12,7 @@ import { targetChangeDirectionValue, targetChangeGroupsForDirection, targetChangeMapIdentity, + targetChangeModeHelp, } from "./target-change-visualization"; function changeMetrics( @@ -153,3 +154,20 @@ describe("target change visualization data", () => { })); }); }); + +test("the comparison mode help names the release the candidate is compared with", () => { + expect(targetChangeModeHelp("Current release").reported).toBe( + "Uses each release's actual target weights and complete target surface. Added targets " + + "contribute candidate error, while removed targets subtract their contribution in the " + + "current release.", + ); + expect(targetChangeModeHelp("Local-area release").reported).toBe( + "Uses each release's actual target weights and complete target surface. Added targets " + + "contribute candidate error, while removed targets subtract their contribution in the " + + "local-area release.", + ); + expect(targetChangeModeHelp("Local-area release").shared).toBe( + "Uses only targets present in both releases and applies the same pooled target weights to " + + "both. Each release still uses its own verified benchmark, scale, cap, and capped error.", + ); +}); diff --git a/frontend/lib/microcosm/target-change-visualization.ts b/frontend/lib/microcosm/target-change-visualization.ts index 9899038c..ef653f6c 100644 --- a/frontend/lib/microcosm/target-change-visualization.ts +++ b/frontend/lib/microcosm/target-change-visualization.ts @@ -11,6 +11,26 @@ import type { export type TargetChangeDirection = "increase" | "reduction"; +/** Help for each comparison mode, naming the release the candidate is compared with. */ +export function targetChangeModeHelp( + comparisonReleaseLabel: string, +): Record { + return { + reported: + "Uses each release's actual target weights and complete target surface. Added targets " + + "contribute candidate error, while removed targets subtract their contribution in the " + + `${sentenceCaseToPhrase(comparisonReleaseLabel)}.`, + shared: + "Uses only targets present in both releases and applies the same pooled target weights to " + + "both. Each release still uses its own verified benchmark, scale, cap, and capped error.", + }; +} + +/** "Local-area release" as it reads mid-sentence: "local-area release". */ +export function sentenceCaseToPhrase(label: string): string { + return label.charAt(0).toLowerCase() + label.slice(1); +} + export function targetChangeMapIdentity(runId: string, releaseId: string): string { return `${runId}:${releaseId}`; } diff --git a/frontend/packages/calibration-tree/src/calibration-tree-manifest.test.ts b/frontend/packages/calibration-tree/src/calibration-tree-manifest.test.ts index 0a8508c5..dc3bf0bf 100644 --- a/frontend/packages/calibration-tree/src/calibration-tree-manifest.test.ts +++ b/frontend/packages/calibration-tree/src/calibration-tree-manifest.test.ts @@ -1,6 +1,7 @@ import { expect, test } from "bun:test"; import { + calibrationBuildsNewestFirst, calibrationTreeManifestEntry, emptyCalibrationTreeManifest, parseCalibrationTreeManifest, @@ -148,3 +149,58 @@ test("release cuts sit beside their line folder's build and keep their folder", }, })).toThrow("invalid release directory"); }); + +test("orders calibration builds by instant across RFC 3339 offsets", () => { + const builds = [ + { + buildArtifactId: "offset-earlier", + createdAt: "2026-01-01T00:30:00+02:00", + updatedAt: "2026-01-01T00:30:00+02:00", + }, + { + buildArtifactId: "utc-later", + createdAt: "2026-01-01T00:00:00Z", + updatedAt: "2026-01-01T00:00:00Z", + }, + { + buildArtifactId: "same-instant-b", + createdAt: null, + updatedAt: "2025-12-31T19:00:00-05:00", + }, + { + buildArtifactId: "same-instant-a", + createdAt: "2026-01-01T00:00:00+00:00", + updatedAt: "2026-01-01T00:00:00+00:00", + }, + ]; + + expect( + calibrationBuildsNewestFirst(builds).map((build) => build.buildArtifactId), + ).toEqual([ + "same-instant-a", + "same-instant-b", + "utc-later", + "offset-earlier", + ]); + expect(builds.map((build) => build.buildArtifactId)).toEqual([ + "offset-earlier", + "utc-later", + "same-instant-b", + "same-instant-a", + ]); +}); + +test("an undated calibration build sorts after every dated one", () => { + const builds = [ + { buildArtifactId: "undated-b", createdAt: null, updatedAt: null }, + { + buildArtifactId: "dated", + createdAt: "2026-10-04T18:57:22Z", + updatedAt: "2026-10-04T18:57:22Z", + }, + { buildArtifactId: "undated-a", createdAt: null, updatedAt: null }, + ]; + expect( + calibrationBuildsNewestFirst(builds).map((build) => build.buildArtifactId), + ).toEqual(["dated", "undated-a", "undated-b"]); +}); diff --git a/frontend/packages/calibration-tree/src/calibration-tree-manifest.ts b/frontend/packages/calibration-tree/src/calibration-tree-manifest.ts index 840b33aa..93222088 100644 --- a/frontend/packages/calibration-tree/src/calibration-tree-manifest.ts +++ b/frontend/packages/calibration-tree/src/calibration-tree-manifest.ts @@ -307,3 +307,31 @@ export function withCalibrationTreeManifestEntry( }, }; } + +interface TimestampedCalibrationBuild { + buildArtifactId: string; + createdAt: string | null; + updatedAt: string | null; +} + +// When a build was published: createdAt, else updatedAt. An undated build +// sorts last rather than wherever NaN lands. +function buildPublishedInstant(build: TimestampedCalibrationBuild): number { + const value = build.createdAt ?? build.updatedAt; + const time = value == null ? Number.NaN : Date.parse(value); + return Number.isFinite(time) ? time : Number.NEGATIVE_INFINITY; +} + +/** + * Build records newest first, by their actual RFC 3339 instant rather than + * timestamp text; builds published at the same instant are ordered by build + * ID. + */ +export function calibrationBuildsNewestFirst< + T extends TimestampedCalibrationBuild, +>(builds: readonly T[]): T[] { + return [...builds].sort((left, right) => + buildPublishedInstant(right) - buildPublishedInstant(left) || + left.buildArtifactId.localeCompare(right.buildArtifactId) + ); +} diff --git a/frontend/packages/calibration-tree/src/countries.ts b/frontend/packages/calibration-tree/src/countries.ts index 515d69c7..94b87deb 100644 --- a/frontend/packages/calibration-tree/src/countries.ts +++ b/frontend/packages/calibration-tree/src/countries.ts @@ -60,6 +60,11 @@ export interface CountryRegistration { repo_env?: string; revision_env?: string; token_env?: string; + // Staging pipelines, by the `pipeline.id` their run manifests record, + // whose candidates are local-area builds. The Staging page compares those + // candidates with the newest published local-area release instead of the + // current (national) release. + local_area_pipelines?: readonly string[]; }; // Other jurisdiction codes a cross-dataset bundle may use for this country. jurisdiction_aliases?: readonly string[]; @@ -120,6 +125,9 @@ export const COUNTRY_REGISTRY = { repo_env: "POPULACE_UK_STAGING_HF_REPO", revision_env: "POPULACE_UK_STAGING_HF_REVISION", token_env: "POPULACE_UK_STAGING_HF_TOKEN", + // Microcosm's rowwise local-area candidates (published as the dense + // release). National attempts run the uk-frs-calibration pipeline. + local_area_pipelines: ["uk-local-candidate"], }, jurisdiction_aliases: ["GB"], }, @@ -201,6 +209,12 @@ export function countryCapabilities(country: MicrocosmCountry): readonly Country return COUNTRY_REGISTRY[country].capabilities; } +// Staging pipeline ids whose candidates are local-area builds; empty for +// countries that register none. +export function stagingLocalAreaPipelines(country: MicrocosmCountry): readonly string[] { + return countryRegistration(country).staging?.local_area_pipelines ?? []; +} + export function hasCapability(country: MicrocosmCountry, capability: CountryCapability): boolean { return countryCapabilities(country).includes(capability); } diff --git a/frontend/packages/calibration-tree/src/staging-artifact.ts b/frontend/packages/calibration-tree/src/staging-artifact.ts index 272f4fca..31717921 100644 --- a/frontend/packages/calibration-tree/src/staging-artifact.ts +++ b/frontend/packages/calibration-tree/src/staging-artifact.ts @@ -10,14 +10,25 @@ import { latestMicrocosmCalibrationSummary, latestMicrocosmTargetDiagnosticsPage, loadCountryRepositoryHashedJson, + loadPointerReleaseId, + loadPublishedCutReleases, loadRelease, + loadReleases, microcosmCountryGeography, microcosmRepo, + type ReleaseEntry, } from "./latest-artifact"; +import { readCalibrationTreeManifest } from "./calibration-tree-blob"; +import { + calibrationBuildsNewestFirst, + type CalibrationTreeManifest, + type CalibrationTreeManifestEntry, +} from "./calibration-tree-manifest"; import type { CalibrationTreeSourceArtifact } from "./calibration-tree-artifact"; import { countryRegistration, resolveRegisteredStagingRepository, + stagingLocalAreaPipelines, type RepositoryEnvironment, type ResolvedRepository, } from "./countries"; @@ -1240,6 +1251,118 @@ export async function loadStagingTargetDiagnostics( return latestMicrocosmTargetDiagnosticsPage(requestUrl, cal); } +// --- which release a staging candidate is compared with ----------------- +// A national candidate is compared with the country's current release, the +// build the dashboard manifest marks latest (for the UK, the one +// latest-national.json names). A local-area candidate calibrates local +// geographies the national release does not, so it is compared with the +// local-area release instead. That release has no pointer file and is never +// promoted, so its most recently published build, as the dashboard manifest +// records it, is used. A run is a local-area candidate when its run +// manifest's pipeline id is one the country registers as local-area: no +// staged receipt or manifest records a dataset role, and the dataset file +// names change between builds. + +export type StagingComparisonBasis = "current_release" | "newest_local_area_release"; + +export interface StagingComparisonRelease { + release_id: string; + basis: StagingComparisonBasis; + /** The run's pipeline builds local-area candidates. */ + local_area_candidate: boolean; +} + +export interface StagingComparisonReleaseDependencies { + runPipelineId(runId: string, country: MicrocosmCountry): Promise; + currentReleaseId(country: MicrocosmCountry): Promise; + newestLocalAreaReleaseId(country: MicrocosmCountry): Promise; +} + +/** The `pipeline.id` a staging run's manifest records, or null without one. */ +export async function stagingRunPipelineId( + runId: string, + revalidate: number, + country: MicrocosmCountry, +): Promise { + assertSafeReleaseId(runId, "run"); + const manifest = await stagingJsonOrNull(`runs/${runId}/run_manifest.json`, revalidate, country); + const pipelineId = asObject(manifest?.pipeline).id; + return typeof pipelineId === "string" && pipelineId.trim() ? pipelineId.trim() : null; +} + +/** + * The most recently published release build whose release is local-area, by + * the dataset roles `releases` carries (read from each release manifest, as + * the release picker reads them) and the publication time the dashboard + * manifest records. + */ +export function newestPublishedLocalAreaRelease( + manifest: CalibrationTreeManifest, + country: MicrocosmCountry, + releases: readonly ReleaseEntry[], +): CalibrationTreeManifestEntry | null { + const localAreaReleases = new Set( + releases.filter((release) => release.is_local_area).map((release) => release.release_id), + ); + const builds = (manifest.countries[country]?.builds ?? []).filter( + (build) => + build.kind === "release" && + build.releaseId != null && + localAreaReleases.has(build.releaseId), + ); + return calibrationBuildsNewestFirst(builds)[0] ?? null; +} + +function defaultStagingComparisonReleaseDependencies( + revalidate: number, +): StagingComparisonReleaseDependencies { + return { + runPipelineId: (runId, country) => stagingRunPipelineId(runId, revalidate, country), + currentReleaseId: async (country) => + (await loadPointerReleaseId(revalidate, country)).release_id, + newestLocalAreaReleaseId: async (country) => { + const { manifest } = await readCalibrationTreeManifest(); + const [folders, tagged] = await Promise.all([ + loadReleases(revalidate, country), + loadPublishedCutReleases(revalidate, country, manifest), + ]); + return newestPublishedLocalAreaRelease(manifest, country, [...folders, ...tagged]) + ?.releaseId ?? null; + }, + }; +} + +export async function resolveStagingComparisonRelease( + runId: string, + country: MicrocosmCountry, + dependencies: StagingComparisonReleaseDependencies = + defaultStagingComparisonReleaseDependencies(300), +): Promise { + const localAreaPipelines = stagingLocalAreaPipelines(country); + const pipelineId = localAreaPipelines.length + ? await dependencies.runPipelineId(runId, country) + : null; + const localAreaCandidate = pipelineId != null && localAreaPipelines.includes(pipelineId); + if (localAreaCandidate) { + const localAreaRelease = await dependencies.newestLocalAreaReleaseId(country); + if (localAreaRelease) { + return { + release_id: localAreaRelease, + basis: "newest_local_area_release", + local_area_candidate: true, + }; + } + } + return { + release_id: await dependencies.currentReleaseId(country), + basis: "current_release", + local_area_candidate: localAreaCandidate, + }; +} + +/** A staging run compared with a release, as `/microcosm/staging/compare` returns it. */ +export type StagingComparison = Awaited>; + export async function loadStagingComparison( runId: string, releaseId: string, diff --git a/frontend/packages/calibration-tree/src/staging-comparison-release.test.ts b/frontend/packages/calibration-tree/src/staging-comparison-release.test.ts new file mode 100644 index 00000000..7e67c2ba --- /dev/null +++ b/frontend/packages/calibration-tree/src/staging-comparison-release.test.ts @@ -0,0 +1,219 @@ +import { expect, test } from "bun:test"; + +import { + emptyCalibrationTreeManifest, + withCalibrationTreeManifestEntry, + type CalibrationTreeManifest, + type CalibrationTreeManifestEntry, +} from "./calibration-tree-manifest"; +import { stagingLocalAreaPipelines } from "./countries"; +import type { ReleaseEntry } from "./latest-artifact"; +import { + newestPublishedLocalAreaRelease, + resolveStagingComparisonRelease, + stagingRunPipelineId, + stagingResolveUrl, + type StagingComparisonReleaseDependencies, +} from "./staging-artifact"; + +// UK shapes as published on 19 Oct 2026. A local-area candidate's staged +// receipt and run manifest record no dataset role, and its dataset file name +// changed between builds (microcosm_uk_2025_local.h5 in September, +// microcosm_uk_2024_25_local.h5 in October). The run manifest's pipeline id, +// which the staging contract validates, is stable: uk-local-candidate for +// local-area candidates and uk-frs-calibration for national attempts. +const NATIONAL_FOLDER = "microcosm-uk-2024-25-national"; +const DENSE_FOLDER = "microcosm-uk-2024-25-dense"; +const NATIONAL_CURRENT = `${NATIONAL_FOLDER}-20261019T083323Z-6cba5d71`; +const NATIONAL_EARLIER = `${NATIONAL_FOLDER}-20261014T214443Z-d12ff9b3`; +const DENSE_EARLIER = `${DENSE_FOLDER}-20261005T101500Z-0d0d0d0d`; +const DENSE_NEWEST = `${DENSE_FOLDER}-20261019T000432Z-ee5c2dbe`; +const JUNE_RELEASE = "populace-uk-2023-dd68c73-4aa4b14-20260619T023711Z"; +const LOCAL_CANDIDATE_RUN = "uk-local-candidate-f100-s42-20261019T000432Z-ee5c2dbe"; +const NATIONAL_ATTEMPT_RUN = "uk-frs-calibration-attempt-20261019T083323Z-6cba5d71"; + +function releaseBuild( + releaseId: string, + createdAt: string, + buildDigit: string, + releaseDirectory?: string, +): CalibrationTreeManifestEntry { + return { + buildArtifactId: buildDigit.repeat(64), + kind: "release", + sourceId: releaseId, + label: releaseId, + releaseId, + ...(releaseDirectory ? { releaseDirectory } : {}), + stagingRunId: null, + hfRepo: "policyengine/populace-uk-private", + hfCommitSha: buildDigit.repeat(40), + treeSchemaVersion: 6, + indexSha256: "e".repeat(64), + indexBytes: 10, + createdAt, + updatedAt: createdAt, + }; +} + +function ukManifest(): CalibrationTreeManifest { + const stagingBuild: CalibrationTreeManifestEntry = { + ...releaseBuild(LOCAL_CANDIDATE_RUN, "2026-10-19T02:05:10.000Z", "9"), + kind: "staging", + releaseId: null, + stagingRunId: LOCAL_CANDIDATE_RUN, + hfRepo: "policyengine/populace-uk-staging", + }; + return [ + releaseBuild(JUNE_RELEASE, "2026-06-19T02:38:38.000Z", "1"), + releaseBuild(NATIONAL_EARLIER, "2026-10-14T22:09:45.000Z", "2", NATIONAL_FOLDER), + releaseBuild(NATIONAL_CURRENT, "2026-10-19T10:21:22.000Z", "3", NATIONAL_FOLDER), + releaseBuild(DENSE_EARLIER, "2026-10-05T10:20:00.000Z", "4", DENSE_FOLDER), + releaseBuild(DENSE_NEWEST, "2026-10-19T07:53:18.000Z", "5", DENSE_FOLDER), + stagingBuild, + ].reduce( + (manifest, entry) => withCalibrationTreeManifestEntry(manifest, "uk", entry), + emptyCalibrationTreeManifest(), + ); +} + +function inventoryEntry(releaseId: string, isLocalArea: boolean): ReleaseEntry { + return { + release_id: releaseId, + date: releaseId, + files: [], + has_calibration: true, + dataset_role: isLocalArea ? "non_default_local_area" : null, + is_default: !isLocalArea, + is_local_area: isLocalArea, + }; +} + +// Roles as the release manifests record them (dataset_role), read the way the +// release picker reads them. +const UK_INVENTORY = [ + inventoryEntry(JUNE_RELEASE, false), + inventoryEntry(NATIONAL_EARLIER, false), + inventoryEntry(NATIONAL_CURRENT, false), + inventoryEntry(DENSE_EARLIER, true), + inventoryEntry(DENSE_NEWEST, true), +]; + +test("only the UK registration names a staging pipeline that builds local-area candidates", () => { + expect(stagingLocalAreaPipelines("uk")).toEqual(["uk-local-candidate"]); + expect(stagingLocalAreaPipelines("us")).toEqual([]); + expect(stagingLocalAreaPipelines("be")).toEqual([]); +}); + +test("the newest local-area release build is the latest one the publisher's manifest records", () => { + expect(newestPublishedLocalAreaRelease(ukManifest(), "uk", UK_INVENTORY)?.releaseId) + .toBe(DENSE_NEWEST); + // Without a local-area role in the inventory, no release qualifies. + expect(newestPublishedLocalAreaRelease( + ukManifest(), + "uk", + UK_INVENTORY.map((entry) => ({ ...entry, is_local_area: false })), + )).toBeNull(); +}); + +function dependencies( + overrides: Partial, +): StagingComparisonReleaseDependencies { + return { + runPipelineId: async () => { + throw new Error("The run's pipeline must not be read."); + }, + currentReleaseId: async () => { + throw new Error("The current release must not be read."); + }, + newestLocalAreaReleaseId: async () => { + throw new Error("The local-area release must not be read."); + }, + ...overrides, + }; +} + +test("US and Belgian staging candidates keep comparing with the current release", async () => { + for (const [country, current] of [ + ["us", "populace-us-2024-spm-receipts-20260923"], + ["be", "microcosm-be-2026-chronicle-3cef97b-20260823T134247Z"], + ] as const) { + await expect(resolveStagingComparisonRelease( + "any-run", + country, + dependencies({ currentReleaseId: async () => current }), + )).resolves.toEqual({ + release_id: current, + basis: "current_release", + local_area_candidate: false, + }); + } +}); + +test("a UK national candidate compares with the current national build", async () => { + await expect(resolveStagingComparisonRelease( + NATIONAL_ATTEMPT_RUN, + "uk", + dependencies({ + runPipelineId: async () => "uk-frs-calibration", + currentReleaseId: async () => NATIONAL_CURRENT, + }), + )).resolves.toEqual({ + release_id: NATIONAL_CURRENT, + basis: "current_release", + local_area_candidate: false, + }); +}); + +test("a UK local-area candidate compares with the newest published local-area build", async () => { + await expect(resolveStagingComparisonRelease( + LOCAL_CANDIDATE_RUN, + "uk", + dependencies({ + runPipelineId: async () => "uk-local-candidate", + newestLocalAreaReleaseId: async () => DENSE_NEWEST, + }), + )).resolves.toEqual({ + release_id: DENSE_NEWEST, + basis: "newest_local_area_release", + local_area_candidate: true, + }); +}); + +test("a UK local-area candidate with no published local-area release falls back to the current release", async () => { + await expect(resolveStagingComparisonRelease( + LOCAL_CANDIDATE_RUN, + "uk", + dependencies({ + runPipelineId: async () => "uk-local-candidate", + newestLocalAreaReleaseId: async () => null, + currentReleaseId: async () => NATIONAL_CURRENT, + }), + )).resolves.toEqual({ + release_id: NATIONAL_CURRENT, + basis: "current_release", + local_area_candidate: true, + }); +}); + +test("a staging run's pipeline id is read from its run manifest", async () => { + const originalFetch = globalThis.fetch; + const manifestUrl = stagingResolveUrl(`runs/${LOCAL_CANDIDATE_RUN}/run_manifest.json`, "uk"); + globalThis.fetch = (async (input: string | URL | Request) => { + const url = String(input); + if (url === manifestUrl) { + return Response.json({ + schema_version: 3, + run_id: LOCAL_CANDIDATE_RUN, + pipeline: { id: "uk-local-candidate", version: "0.1.0" }, + }); + } + return new Response(null, { status: 404 }); + }) as typeof fetch; + try { + await expect(stagingRunPipelineId(LOCAL_CANDIDATE_RUN, 0, "uk")).resolves.toBe("uk-local-candidate"); + await expect(stagingRunPipelineId("missing-run", 0, "uk")).resolves.toBeNull(); + } finally { + globalThis.fetch = originalFetch; + } +});