From 5ad86340f26754277f83b7ddd93da5ab0be5cc99 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 20:38:42 +0200 Subject: [PATCH 1/2] chore(ci): require upstreamImage in catalog sync and skip superseded hotfixes - RevisionIo's manifest and provenance fetchers are required, and every refreshed pin carries upstreamImage, so a written Artifacts.ts always typechecks. - planSlimUpdates skips a line's hotfix when that line upgrades, warning that manual --release can pin it alone, instead of opening a hotfix PR that conflicts once the upgrade merges. - The sync script's mode docs and the slim-release-published workflow header move into ADR 0026; provenance and narration comments are trimmed across the slim catalog files. - supabase services listing tests run against the real catalog again. --- .../scripts/sync-artifacts-catalog.test.ts | 103 +++++++---- .github/scripts/sync-artifacts-catalog.ts | 168 ++++++------------ .github/workflows/slim-release-published.yml | 70 ++------ ...nder-service-dockerfile.rules.unit.test.ts | 6 +- .../render-service-dockerfile.unit.test.ts | 3 +- .../commands/start/services/vector.service.ts | 11 +- .../services/services.shared.unit.test.ts | 130 ++++++++++---- apps/cli/src/shared/services/slim-images.ts | 39 +--- .../shared/services/slim-images.unit.test.ts | 10 +- apps/cli/tests/helpers/slim-images.ts | 9 +- docs/adr/0026-slim-artifact-mirrors.md | 80 +++++++-- packages/stack/src/services/Vector.ts | 13 +- 12 files changed, 316 insertions(+), 326 deletions(-) diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index 1a266d12be..ebc4d8962e 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -51,6 +51,25 @@ function matchingS3( return async (url) => byUrl.get(url); } +/** A `fetchManifest` returning a derived service's `upstream_image`, for every native target. */ +const manifestWithUpstreamImage = + (image: string): RevisionIo["fetchManifest"] => + async () => + JSON.stringify({ upstream_image: image }); + +/** A `fetchProvenance` returning a mirrored service's `source`. */ +const provenanceWithSource = + (source: string): RevisionIo["fetchProvenance"] => + async () => + JSON.stringify({ source }); + +const unusedFetchManifest: RevisionIo["fetchManifest"] = async () => { + throw new Error("fetchManifest should not be called for this service"); +}; +const unusedFetchProvenance: RevisionIo["fetchProvenance"] = async () => { + throw new Error("fetchProvenance should not be called for this service"); +}; + /** * Runs the repo's pinned `oxfmt` binary over `source`, the way `slim-release-published.yml` * formats the catalog after every write, so parsing tests exercise real formatter output @@ -281,22 +300,13 @@ describe("planSlimUpdates", () => { ]); }); - test("both a hotfix and an upgrade can be planned in the same run", () => { - const { updates } = planSlimUpdates(plannerFixture, "postgrest", [ + test("an upgrade on a line suppresses that line's hotfix, warning about the skipped release", () => { + const { updates, warnings } = planSlimUpdates(plannerFixture, "postgrest", [ "postgrest-v16.2-r1", "postgrest-v16.4-r0", ]); expect(updates).toEqual([ - { - kind: "hotfix", - line: undefined, - branch: "slim-hotfix/postgrest", - title: "chore(stack): pin postgrest v16.2-r1", - fromRelease: "v16.2-r0", - toUpstream: "v16.2", - toRelease: "v16.2-r1", - }, { kind: "upgrade", line: undefined, @@ -307,6 +317,9 @@ describe("planSlimUpdates", () => { toRelease: "v16.4-r0", }, ]); + expect(warnings).toEqual([ + "::warning ::postgrest v16.2-r1 hotfix skipped because this line is upgrading to v16.4-r0; run --service postgrest --release v16.2-r1 to pin the hotfix alone.", + ]); }); test("an older committed upstream is a backlog republish: it plans nothing", () => { @@ -326,7 +339,7 @@ describe("planSlimUpdates", () => { ]); // The ignored major-18 tag is warned about, but doesn't block the two valid updates - // alongside it (P1: a warning must never corrupt or swallow the plan). + // alongside it: a warning never corrupts or swallows the plan. expect(warnings).toEqual([ "::warning ::postgres 18.0.0.001 is not on a release line packages/stack/src/Artifacts.ts carries for it; ignoring postgres-18.0.0.001-r0.", ]); @@ -570,7 +583,7 @@ describe("planUpdatesForService (the plan-updates transport's IO seam)", () => { }); }); -describe("waitForExpectedRelease (item A's eventual-consistency wait, call counts a subprocess can't assert)", () => { +describe("waitForExpectedRelease", () => { test("visible on the first listing: returns immediately without waiting", async () => { let calls = 0; const waits: number[] = []; @@ -645,7 +658,7 @@ describe("waitForExpectedRelease (item A's eventual-consistency wait, call count }); describe("runPlanUpdates (the actual plan-updates CLI mode, not just the pure planner)", () => { - test("an ignored tag, a valid upgrade, and the expected dispatched release: --output gets exactly the valid record, the warning reaches stdout, and the process exits 0", async () => { + test("an ignored tag, a valid upgrade, and the expected dispatched release: --output gets only the valid record, the warning reaches stdout, and the process exits 0", async () => { // Real catalog, real "postgrest" pin — a local fixture server stands in for the releases API // (`SLIM_SERVICES_RELEASES_API`), so this exercises the real subprocess: the CLI's argv // parsing, `--expect-release`'s visibility wait, the network lister, and the file/stdout @@ -794,7 +807,7 @@ describe("runPlanUpdates (the actual plan-updates CLI mode, not just the pure pl } }); - test("a newline-bearing --format is rejected, producing exactly one ::error :: line (not JSON.stringify'd; the boundary encoder is what protects it)", async () => { + test("a newline-bearing --format is rejected, producing a single ::error :: line (not JSON.stringify'd; the boundary encoder is what protects it)", async () => { const dir = await mkdtemp(join(tmpdir(), "plan-updates-bad-format-")); const outputPath = join(dir, "slim-updates.tsv"); try { @@ -863,6 +876,8 @@ describe("refreshCatalogPin", () => { fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r1", digests), imageDigest: async () => digest("h"), s3Sha256: matchingS3("analytics", "v1.50.9-r1", digests), + fetchManifest: manifestWithUpstreamImage("supabase/logflare:1.50.9"), + fetchProvenance: unusedFetchProvenance, }; const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); @@ -878,6 +893,7 @@ describe("refreshCatalogPin", () => { expect(result.source).toContain( `image: "ghcr.io/supabase/cli/analytics:v1.50.9-r1@${digest("h")}"`, ); + expect(result.source).toContain('upstreamImage: "supabase/logflare:1.50.9"'); }); test("a resolved pin survives real formatting and can be refreshed again", async () => { @@ -887,6 +903,8 @@ describe("refreshCatalogPin", () => { fetchChecksums: async () => checksumsFor("postgrest", "v16.2-r0", first), imageDigest: async () => digest("j"), s3Sha256: matchingS3("postgrest", "v16.2-r0", first), + fetchManifest: manifestWithUpstreamImage("postgrest/postgrest:v16.2"), + fetchProvenance: unusedFetchProvenance, }; const written = await refreshCatalogPin({ catalog: fixture, @@ -906,6 +924,8 @@ describe("refreshCatalogPin", () => { fetchChecksums: async () => checksumsFor("postgrest", "v16.2-r1", second), imageDigest: async () => digest("l"), s3Sha256: matchingS3("postgrest", "v16.2-r1", second), + fetchManifest: manifestWithUpstreamImage("postgrest/postgrest:v16.2"), + fetchProvenance: unusedFetchProvenance, }; const refreshed = await refreshCatalogPin({ catalog: formatted, @@ -922,6 +942,7 @@ describe("refreshCatalogPin", () => { }); expect(refreshed.source).toContain("revision: 1"); expect(refreshed.source).toContain(second["darwin-arm64"].archive); + expect(refreshed.source).toContain('upstreamImage: "postgrest/postgrest:v16.2"'); }); test("refreshes the Postgres 15 additional pin, including after formatting", async () => { @@ -931,6 +952,8 @@ describe("refreshCatalogPin", () => { fetchChecksums: async () => checksumsFor("postgres", "15.14.1.168-r0", digests), imageDigest: async () => digest("n"), s3Sha256: matchingS3("postgres", "15.14.1.168-r0", digests), + fetchManifest: manifestWithUpstreamImage("supabase/postgres:15.14.1.168"), + fetchProvenance: unusedFetchProvenance, }; const written = await refreshCatalogPin({ @@ -956,6 +979,8 @@ describe("refreshCatalogPin", () => { fetchChecksums: async () => checksumsFor("postgres", "15.14.1.168-r1", nextDigests), imageDigest: async () => digest("p"), s3Sha256: matchingS3("postgres", "15.14.1.168-r1", nextDigests), + fetchManifest: manifestWithUpstreamImage("supabase/postgres:15.14.1.168"), + fetchProvenance: unusedFetchProvenance, }; const refreshed = await refreshCatalogPin({ @@ -974,6 +999,7 @@ describe("refreshCatalogPin", () => { }); expect(refreshed.source).toContain(postgres17Image); expect(refreshed.source).toContain(nextDigests["linux-arm64"].manifest); + expect(refreshed.source).toContain('upstreamImage: "supabase/postgres:15.14.1.168"'); }); test("moves the Postgres 15 additional pin to a new upstream version, updating its key", async () => { @@ -983,6 +1009,8 @@ describe("refreshCatalogPin", () => { fetchChecksums: async () => checksumsFor("postgres", "15.19.0.002-r0", digests), imageDigest: async () => digest("v"), s3Sha256: matchingS3("postgres", "15.19.0.002-r0", digests), + fetchManifest: manifestWithUpstreamImage("supabase/postgres:15.19.0.002"), + fetchProvenance: unusedFetchProvenance, }; const written = await refreshCatalogPin({ @@ -1012,6 +1040,8 @@ describe("refreshCatalogPin", () => { fetchChecksums: async () => checksumsFor("postgres", "15.19.0.002-r1", nextDigests), imageDigest: async () => digest("x"), s3Sha256: matchingS3("postgres", "15.19.0.002-r1", nextDigests), + fetchManifest: manifestWithUpstreamImage("supabase/postgres:15.19.0.002"), + fetchProvenance: unusedFetchProvenance, }; const refreshed = await refreshCatalogPin({ catalog: formatted, @@ -1028,6 +1058,7 @@ describe("refreshCatalogPin", () => { target: "additional", }); expect(refreshed.source).toContain(nextDigests["linux-arm64"].manifest); + expect(refreshed.source).toContain('upstreamImage: "supabase/postgres:15.19.0.002"'); }); }); @@ -1054,6 +1085,8 @@ describe("resolveRevisionPin waits for the S3 mirror", () => { } return present(url); }, + fetchManifest: unusedFetchManifest, + fetchProvenance: unusedFetchProvenance, wait: async (ms) => { waits.push(ms); }, @@ -1075,6 +1108,8 @@ describe("resolveRevisionPin waits for the S3 mirror", () => { fetchChecksums: async () => checksumsFor("postgrest", releaseVersion, digests), imageDigest: async () => digest("s3-mismatch-digest"), s3Sha256: async () => hex("wrong-bytes"), + fetchManifest: unusedFetchManifest, + fetchProvenance: unusedFetchProvenance, wait: async (ms) => { waits.push(ms); }, @@ -1097,6 +1132,8 @@ describe("resolveRevisionPin waits for the S3 mirror", () => { fetchChecksums: async () => checksumsFor("postgrest", releaseVersion, digests), imageDigest: async () => digest("s3-never-digest"), s3Sha256: async () => undefined, + fetchManifest: unusedFetchManifest, + fetchProvenance: unusedFetchProvenance, wait: async (ms) => { waits.push(ms); }, @@ -1113,13 +1150,15 @@ describe("resolveRevisionPin waits for the S3 mirror", () => { }); describe("refreshCatalogPin --release", () => { - test("pins exactly the requested committed release, not the highest one", async () => { + test("pins the requested committed release, not the highest one", async () => { const digests = nativeDigests("release-0"); const io: RevisionIo = { listReleaseTags: async () => ["postgrest-v16.2-r0", "postgrest-v16.2-r1"], fetchChecksums: async () => checksumsFor("postgrest", "v16.2-r0", digests), imageDigest: async () => digest("release-digest-0"), s3Sha256: matchingS3("postgrest", "v16.2-r0", digests), + fetchManifest: manifestWithUpstreamImage("postgrest/postgrest:v16.2"), + fetchProvenance: unusedFetchProvenance, }; const result = await refreshCatalogPin({ @@ -1144,6 +1183,8 @@ describe("refreshCatalogPin --release", () => { fetchChecksums: async () => undefined, imageDigest: async () => undefined, s3Sha256: async () => undefined, + fetchManifest: unusedFetchManifest, + fetchProvenance: unusedFetchProvenance, }; await expect( @@ -1157,6 +1198,8 @@ describe("refreshCatalogPin --release", () => { fetchChecksums: async () => undefined, imageDigest: async () => undefined, s3Sha256: async () => undefined, + fetchManifest: unusedFetchManifest, + fetchProvenance: unusedFetchProvenance, }; await expect( @@ -1180,6 +1223,7 @@ describe("refreshCatalogPin backfills upstreamImage", () => { imageDigest: async () => digest("z"), s3Sha256: matchingS3("analytics", "v1.50.9-r1", digests), fetchManifest: async () => JSON.stringify({ upstream_image: "supabase/logflare:1.50.9" }), + fetchProvenance: unusedFetchProvenance, }; const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); @@ -1195,6 +1239,7 @@ describe("refreshCatalogPin backfills upstreamImage", () => { imageDigest: async () => digest("ab"), s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), fetchManifest: async () => JSON.stringify({ source_image: "supabase/logflare:1.50.9" }), + fetchProvenance: unusedFetchProvenance, }; const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); @@ -1211,6 +1256,7 @@ describe("refreshCatalogPin backfills upstreamImage", () => { s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), fetchManifest: async () => JSON.stringify({ upstream_image: "docker.io/supabase/logflare:1.50.9@sha256:deadbeef" }), + fetchProvenance: unusedFetchProvenance, }; const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); @@ -1230,6 +1276,7 @@ describe("refreshCatalogPin backfills upstreamImage", () => { call += 1; return JSON.stringify({ upstream_image: `supabase/logflare:1.50.${call}` }); }, + fetchProvenance: unusedFetchProvenance, }; await expect(refreshCatalogPin({ catalog: fixture, service: "analytics", io })).rejects.toThrow( @@ -1244,8 +1291,8 @@ describe("refreshCatalogPin backfills upstreamImage", () => { fetchChecksums: async () => checksumsFor("vector", "0.53.0-r0", digests), imageDigest: async () => digest("ah"), s3Sha256: matchingS3("vector", "0.53.0-r0", digests), - fetchProvenance: async () => - JSON.stringify({ source: "docker.io/timberio/vector:0.53.0-alpine" }), + fetchManifest: unusedFetchManifest, + fetchProvenance: provenanceWithSource("docker.io/timberio/vector:0.53.0-alpine"), }; const result = await refreshCatalogPin({ catalog: vectorFixture, service: "vector", io }); @@ -1253,20 +1300,6 @@ describe("refreshCatalogPin backfills upstreamImage", () => { expect(result.source).toContain('upstreamImage: "timberio/vector:0.53.0-alpine"'); }); - test("leaves upstreamImage absent when io has no manifest/provenance fetchers", async () => { - const digests = nativeDigests("ai"); - const io: RevisionIo = { - listReleaseTags: async () => ["analytics-v1.50.9-r0"], - fetchChecksums: async () => checksumsFor("analytics", "v1.50.9-r0", digests), - imageDigest: async () => digest("aj"), - s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), - }; - - const result = await refreshCatalogPin({ catalog: fixture, service: "analytics", io }); - - expect(result.source).not.toContain("upstreamImage"); - }); - test("rejects a manifest upstream_image carrying a quote or template expression, writing nothing", async () => { const digests = nativeDigests("ak"); const io: RevisionIo = { @@ -1276,6 +1309,7 @@ describe("refreshCatalogPin backfills upstreamImage", () => { s3Sha256: matchingS3("analytics", "v1.50.9-r0", digests), fetchManifest: async () => JSON.stringify({ upstream_image: 'supabase/logflare:1.50.9"] }; import("evil"); //' }), + fetchProvenance: unusedFetchProvenance, }; await expect(refreshCatalogPin({ catalog: fixture, service: "analytics", io })).rejects.toThrow( @@ -1298,6 +1332,8 @@ describe("against the real catalog", () => { fetchChecksums: async () => checksumsFor("auth", `${pinnedVersion}-r0`, first), imageDigest: async () => digest("r"), s3Sha256: matchingS3("auth", `${pinnedVersion}-r0`, first), + fetchManifest: manifestWithUpstreamImage(`supabase/gotrue:${pinnedVersion}`), + fetchProvenance: unusedFetchProvenance, }; const written = await refreshCatalogPin({ catalog, service: "auth", io: firstIo }); expect(written.update?.revision).toBe(0); @@ -1311,6 +1347,8 @@ describe("against the real catalog", () => { fetchChecksums: async () => checksumsFor("auth", `${pinnedVersion}-r1`, second), imageDigest: async () => digest("t"), s3Sha256: matchingS3("auth", `${pinnedVersion}-r1`, second), + fetchManifest: manifestWithUpstreamImage(`supabase/gotrue:${pinnedVersion}`), + fetchProvenance: unusedFetchProvenance, }; const refreshed = await refreshCatalogPin({ catalog: formatted, @@ -1327,5 +1365,6 @@ describe("against the real catalog", () => { }); expect(refreshed.source).toContain("revision: 1"); expect(refreshed.source).toContain(second["darwin-arm64"].manifest); + expect(refreshed.source).toContain(`upstreamImage: "supabase/gotrue:${pinnedVersion}"`); }); }); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index 0ec85a6b02..e568e07ab4 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -1,43 +1,6 @@ /** - * Pins `packages/stack/src/Artifacts.ts` to committed `supabase/slim-services` - * revisions (`-r`). Every entry is pinned by content: the GHCR - * image digest, plus an archive and manifest sha256 per native target. - * - * The catalog is the single version table (supabase/cli#6883): the Dockerfile's - * slim-capable lines are a generated view of it - * (`apps/cli/scripts/render-service-dockerfile.ts`), never the other way - * around. Updates land through two modes: - * - * Manual mode: refreshes one catalog entry, either to a specific committed - * release (`--release`) or to the highest committed revision of a given - * upstream version, or of its currently pinned upstream version when neither - * is given. `--upstream` and `--release` are mutually exclusive. - * - * bun .github/scripts/sync-artifacts-catalog.ts --service [--upstream | --release -r] - * - * Plan-updates mode (used by the `slim-release-published` dispatch workflow): - * lists a service's committed slim-services releases and computes, per - * release line, the hotfix and/or upgrade a workflow should apply. Pure - * planning: `planSlimUpdates` takes the release tag list as an argument, - * makes no network calls, and never logs — it returns `{ updates, warnings }`. - * Records go only to `--output ` (never stdout); warnings print to - * stdout as `::warning ::…` lines, so the two channels can't corrupt one - * another when a caller redirects stdout separately from the records file. - * - * bun .github/scripts/sync-artifacts-catalog.ts plan-updates --service \ - * --output [--format lines] [--expect-release -r] - * - * `--expect-release` handles the releases API lagging behind the dispatch that triggered this - * run: before planning, the listed committed tags must include `-`, or - * this mode re-lists a bounded number of times before giving up (`waitForExpectedRelease`). - * - * Validate-payload mode (used by the same workflow, before anything else): - * checks an untrusted `slim-release-published` dispatch payload against - * anchored charsets and prints it back as `key=value` lines, so a value that - * fails validation is never written to `$GITHUB_OUTPUT` in the first place. - * - * bun .github/scripts/sync-artifacts-catalog.ts validate-payload --service \ - * --upstream --revision --release + * Pins `packages/stack/src/Artifacts.ts` to committed `supabase/slim-services` revisions. See + * `docs/adr/0026-slim-artifact-mirrors.md` for the catalog model and this script's three modes. */ import { @@ -54,7 +17,7 @@ export const CATALOG_PATH = "packages/stack/src/Artifacts.ts"; const SLIM_IMAGE_PREFIX = `${SOURCE_REGISTRY}/`; -/** The three native targets the catalog pins per revision. Kept self-contained; see module docs. */ +/** Native targets the catalog pins per revision, in `Artifacts.ts`'s serialization order. */ const NATIVE_TARGETS = ["darwin-arm64", "linux-amd64", "linux-arm64"] as const; type NativeTargetName = (typeof NATIVE_TARGETS)[number]; @@ -67,7 +30,7 @@ const RELEASE_DOWNLOAD_BASE = "https://github.com/supabase/slim-services/release /** Bounded retry for `waitForExpectedRelease`: 6 attempts, 10s apart, by default. */ const EXPECT_RELEASE_ATTEMPTS = 6; const DEFAULT_EXPECT_RELEASE_INTERVAL_MS = 10_000; -/** Bounded retry for the S3-mirror wait in `resolveRevisionPin`: ~10 min, covering a native upload of all three targets (`mirror-slim-image.yml`'s `upload-natives-s3`). */ +/** ~10 min for `resolveRevisionPin`'s S3-mirror wait: `upload-natives-s3` for all three targets. */ const S3_WAIT_ATTEMPTS = 20; const DEFAULT_S3_WAIT_INTERVAL_MS = 30_000; const MAX_TIMER_DELAY_MS = 2 ** 31 - 1; // setTimeout's own ceiling. @@ -109,12 +72,7 @@ type BoundedWaitOutcome = | { readonly status: "timed-out" } | { readonly status: "failed"; readonly message: string }; -/** - * Generic bounded retry-with-wait: `waitForExpectedRelease`'s release-visibility wait and - * `resolveRevisionPin`'s S3-mirror wait both build on this. Calls `check` up to `attempts` - * times, `wait`-ing `intervalMs` between, until it reports `"done"` or an unrecoverable - * `"failed"` (never waited out). Exhausting every attempt on `"retry"` yields `"timed-out"`. - */ +/** Calls `check` up to `attempts` times, waiting between, until it reports `done` or `failed`. */ async function boundedWait( check: () => Promise>, wait: (ms: number) => Promise, @@ -196,21 +154,19 @@ export interface CatalogPinUpdate { readonly target: "default" | "additional"; } -/** Content pin for one resolved `-r` revision, ready to serialize into the catalog. */ +/** Content pin for one resolved `-r` revision, without its `upstreamImage`. */ interface ResolvedPin { readonly upstreamVersion: string; readonly revision: number; readonly image: string; - /** - * The pin's `ArtifactPin.upstreamImage`. Populated by manual mode (`refreshCatalogPin`) via - * `resolveUpstreamImage` below, when `io` carries `fetchManifest`/`fetchProvenance`. - */ - readonly upstreamImage?: string; readonly natives: Readonly< Record >; } +/** A `ResolvedPin` plus its `ArtifactPin.upstreamImage`, ready to serialize into the catalog. */ +type CatalogPin = ResolvedPin & { readonly upstreamImage: string }; + export type RevisionResolution = | { readonly status: "resolved"; readonly pin: ResolvedPin } | { @@ -234,20 +190,18 @@ export interface RevisionIo { /** * Raw contents of a native target's `.manifest.json` release asset, for a derived service's * `upstream_image` (or `source_image` on an image-derived build, e.g. postgrest's Linux - * targets). Optional: only manual mode's `upstreamImage` backfill (`resolveUpstreamImage`) - * calls this, so a test `io` that doesn't exercise that path can omit it. + * targets). */ - readonly fetchManifest?: ( + readonly fetchManifest: ( service: string, releaseVersion: string, target: NativeTargetName, ) => Promise; /** * Raw contents of a mirrored service's `-.oci-provenance.json` - * release asset, whose `source` field is the mirrored upstream image. Optional for the same - * reason as `fetchManifest`. + * release asset, whose `source` field is the mirrored upstream image. */ - readonly fetchProvenance?: ( + readonly fetchProvenance: ( service: string, releaseVersion: string, upstreamVersion: string, @@ -266,9 +220,8 @@ function desiredImage(service: string, releaseVersion: string, digest: string): * manifest digest, and every native target's archive and manifest sha256, cross-checked against * the S3 mirror copy. * - * With `requiredRevision` given, that exact revision must already be committed — this is what - * pins exactly a planned release (`--release`), never "highest at apply time". Without it, the - * highest committed revision of `upstream` is used (`--upstream`, and the hotfix/upgrade default). + * With `requiredRevision` given (`--release`), that revision must already be committed; it is + * never "highest at apply time". Without it, the highest committed revision of `upstream` is used. */ export async function resolveRevisionPin( service: string, @@ -449,7 +402,7 @@ function validateUpstreamImage(image: string, context: string): string { * manifest `upstream_image` (falling back to `source_image` for an image-derived build, e.g. * postgrest's Linux targets) — cross-checked across every native target, so a derived service * whose manifests disagree fails instead of silently picking one. Only manual mode - * (`refreshCatalogPin`) calls this; `io` must carry `fetchManifest`/`fetchProvenance`. + * (`refreshCatalogPin`) calls this. */ async function resolveUpstreamImage( service: string, @@ -458,11 +411,6 @@ async function resolveUpstreamImage( io: RevisionIo, ): Promise { if (MIRROR_MODE_SOURCE_SERVICES.has(service)) { - if (io.fetchProvenance === undefined) { - throw new InvalidPayloadError( - `${service} needs an io.fetchProvenance to resolve upstreamImage.`, - ); - } const provenance = await io.fetchProvenance(service, releaseVersion, upstreamVersion); if (provenance === undefined) { throw new InvalidPayloadError(`${service}-${releaseVersion} has no oci-provenance asset.`); @@ -476,9 +424,6 @@ async function resolveUpstreamImage( return validateUpstreamImage(normalizeUpstreamImage(source), `${service}-${releaseVersion}`); } - if (io.fetchManifest === undefined) { - throw new InvalidPayloadError(`${service} needs an io.fetchManifest to resolve upstreamImage.`); - } const values = new Set(); for (const target of NATIVE_TARGETS) { const manifest = await io.fetchManifest(service, releaseVersion, target); @@ -513,14 +458,12 @@ async function resolveUpstreamImage( * text is written straight into TypeScript source that later gets imported, so an unescaped * quote or template expression in any field (release metadata included) would inject code. */ -function serializePin(pin: ResolvedPin): string { +function serializePin(pin: CatalogPin): string { const natives = NATIVE_TARGETS.map((target) => { const native = pin.natives[target]; return `${JSON.stringify(target)}: { archive: ${JSON.stringify(native.archive)}, manifest: ${JSON.stringify(native.manifest)} }`; }).join(", "); - const upstreamImage = - pin.upstreamImage === undefined ? "" : ` upstreamImage: ${JSON.stringify(pin.upstreamImage)},`; - return `{ upstreamVersion: ${JSON.stringify(pin.upstreamVersion)}, revision: ${pin.revision}, image: ${JSON.stringify(pin.image)},${upstreamImage} natives: { ${natives} } }`; + return `{ upstreamVersion: ${JSON.stringify(pin.upstreamVersion)}, revision: ${pin.revision}, image: ${JSON.stringify(pin.image)}, upstreamImage: ${JSON.stringify(pin.upstreamImage)}, natives: { ${natives} } }`; } interface PinSpan { @@ -568,7 +511,7 @@ const PIN_UPSTREAM_VERSION = /upstreamVersion:\s*"([^"]+)"/; const PIN_REVISION = /revision:\s*(\d+)/; /** - * Matches a resolved `ArtifactPin` object literal starting exactly at `index`. The span is found + * Matches a resolved `ArtifactPin` object literal that starts at `index`. The span is found * by bracket balance, then the version is pulled out with a loose field search — so a formatter's * whitespace, line breaks, trailing commas, or property order never break matching, only the * literal shape itself would. @@ -729,7 +672,7 @@ const normalizeText = (text: string): string => text.replace(/\s+/g, " ").trim() function writePin( source: string, entry: Extract, - pin: ResolvedPin, + pin: CatalogPin, ): { readonly source: string; readonly changed: boolean } { const desired = serializePin(pin); const current = source.slice(entry.span.start, entry.span.end); @@ -821,24 +764,10 @@ export interface PlanSlimUpdatesResult { const SINGLE_LINE_KEY = "*"; /** - * The hotfix and/or upgrade a `slim-release-published` run should apply for `service`, computed - * against `catalog`'s current pins and `releaseTags` (every committed — published, non-draft — - * slim-services release tag; the caller filters drafts before calling this). Pure: no network, no - * file I/O, no logging — every diagnostic comes back in `warnings` instead, so a caller (the CLI, - * or a test) decides where it goes. This matters because `plan-updates` writes `updates` straight - * into a file the workflow parses as records; a `console.log`'d warning on the same stdout the - * workflow captures would corrupt that file instead of just being informational. - * - * Per release line (the default pin's line, plus one per additional pin — only postgres - * has more than one) a **hotfix** fires when the pinned upstream has a committed revision higher - * than the pinned one; an **upgrade** fires when the newest committed upstream on the line is - * newer than the pinned one (ties, e.g. two Studio builds dated the same day, are not newer). - * Both can fire in the same run. A service with no additional pins has exactly one line and - * accepts any comparable upstream on it — a Studio year rollover or a postgrest major bump is - * the same line moving forward, not a different one, so it is never filtered by `releaseLine`. - * Only a service that does carry additional pins (postgres) filters a release tag to the line its - * `releaseLine` names; a tag on no such line, or whose version isn't comparable, is warned about - * and ignored — it never causes a plan-updates run to fail. + * The upgrade or hotfix each of `service`'s release lines should get, planned against `catalog`'s + * pins and the committed (non-draft) `releaseTags`; the rules are in ADR 0026, "Hotfix and upgrade + * pickup". Never logs: `plan-updates` writes `updates` to a file the workflow parses as records, + * so diagnostics come back in `warnings`. */ export function planSlimUpdates( catalog: string, @@ -907,14 +836,8 @@ export function planSlimUpdates( const sameUpstreamRevisions = candidates .filter((candidate) => candidate.upstream === pinned.upstream) .map((candidate) => candidate.revision); - if (sameUpstreamRevisions.length > 0) { - const highest = Math.max(...sameUpstreamRevisions); - if (highest > pinned.revision) { - updates.push( - buildUpdate("hotfix", service, line, hasLines, pinned, pinned.upstream, highest), - ); - } - } + const hotfixRevision = + sameUpstreamRevisions.length > 0 ? Math.max(...sameUpstreamRevisions) : undefined; const highestRevisionByUpstream = new Map(); for (const candidate of candidates) { @@ -940,9 +863,23 @@ export function planSlimUpdates( bestUpstream = upstream; } } + if (bestUpstream !== undefined && compareVersions(bestUpstream, pinned.upstream) === 1) { const revision = highestRevisionByUpstream.get(bestUpstream) as number; updates.push(buildUpdate("upgrade", service, line, hasLines, pinned, bestUpstream, revision)); + if (hotfixRevision !== undefined && hotfixRevision > pinned.revision) { + const skippedRelease = `${pinned.upstream}-r${hotfixRevision}`; + warnings.push( + workflowCommand( + "warning", + `${service} ${skippedRelease} hotfix skipped because this line is upgrading to ${bestUpstream}-r${revision}; run --service ${service} --release ${skippedRelease} to pin the hotfix alone.`, + ), + ); + } + } else if (hotfixRevision !== undefined && hotfixRevision > pinned.revision) { + updates.push( + buildUpdate("hotfix", service, line, hasLines, pinned, pinned.upstream, hotfixRevision), + ); } } @@ -1009,7 +946,7 @@ export interface CatalogRefreshResult { } /** - * Refreshes one catalog entry: to exactly the committed release named by `release` (`-r`, + * Refreshes one catalog entry: to the committed release named by `release` (`-r`, * required to already be committed — never "highest at apply time"), or to the highest committed * revision of `upstream` (or, when both are omitted, of the entry's currently pinned upstream * version). `upstream` and `release` are mutually exclusive. Pure aside from `io`: callers own @@ -1059,20 +996,15 @@ export async function refreshCatalogPin(input: { if (resolution.status !== "resolved") { throw new InvalidPayloadError(resolution.message); } - // Manual mode also backfills `upstreamImage`, so a plain `io` (most tests) can still exercise - // revision resolution without stubbing the extra fetchers. - const pin = - input.io.fetchManifest === undefined && input.io.fetchProvenance === undefined - ? resolution.pin - : { - ...resolution.pin, - upstreamImage: await resolveUpstreamImage( - input.service, - `${resolution.pin.upstreamVersion}-r${resolution.pin.revision}`, - resolvedUpstream, - input.io, - ), - }; + const pin: CatalogPin = { + ...resolution.pin, + upstreamImage: await resolveUpstreamImage( + input.service, + `${resolution.pin.upstreamVersion}-r${resolution.pin.revision}`, + resolvedUpstream, + input.io, + ), + }; const written = writePin(input.catalog, entry, pin); if (!written.changed) return { source: input.catalog }; return { diff --git a/.github/workflows/slim-release-published.yml b/.github/workflows/slim-release-published.yml index 9e90d9a362..2307ab2b01 100644 --- a/.github/workflows/slim-release-published.yml +++ b/.github/workflows/slim-release-published.yml @@ -1,23 +1,7 @@ name: Slim Release Published -# supabase/slim-services sends this dispatch after it mints an immutable release -# `--r`. The stack catalog -# (packages/stack/src/Artifacts.ts) is the single version table for the service; this workflow -# treats the dispatch as a trigger and reconciles the catalog against every committed -# (published, non-draft) `-...-r` release, opening or updating one pull request per -# hotfix and/or upgrade it finds (`.github/scripts/sync-artifacts-catalog.ts`'s `planSlimUpdates`). -# -# Plan and apply run from the same checkout of the default branch, in a single job: a re-run -# always recomputes from the latest develop, so a stale plan can never be applied, and a -# superseded PR's branch is rewritten in place (force-pushed) instead of racing a fresh one. A -# backlog republish of an older upstream version naturally plans nothing. -# -# The payload arrives with whatever authority holds the dispatch token, so it is treated as -# untrusted: fields are pattern- and shape-checked before use -# (`validateSlimReleasePublishedPayload`), and never interpolated directly into a `run:` script -# (only passed through `env:`). Release tag names come from an external API too, so every value -# `planSlimUpdates` emits is re-validated against the same anchored patterns before it can reach a -# branch name or PR title. +# Reconciles the stack catalog against supabase/slim-services releases. See +# docs/adr/0026-slim-artifact-mirrors.md, "Hotfix and upgrade pickup". on: repository_dispatch: @@ -46,9 +30,7 @@ jobs: ref: ${{ github.event.repository.default_branch }} persist-credentials: false - # Installs the locked workspace dependencies the scripts need: `Artifacts.ts` imports - # `effect`, and `render-service-dockerfile.ts` imports `@supabase/stack/internal/artifacts`. - # Replaces the bare mise step the pre-single-table version of this workflow used. + # `Artifacts.ts` and `render-service-dockerfile.ts` import locked workspace dependencies. - name: Setup uses: ./.github/actions/setup with: @@ -58,13 +40,7 @@ jobs: id: base run: echo "sha=$(git rev-parse HEAD)" >>"$GITHUB_OUTPUT" - # Delegates to `validateSlimReleasePublishedPayload` (sync-artifacts-catalog.ts) so every - # field — including `upstream_version` and `release_version`, not just `service` and - # `revision` — is checked against an anchored charset before it is ever written to - # `$GITHUB_OUTPUT`, a branch name, or a PR title. A value that fails validation makes the - # script exit non-zero and print a single `::error ::…` line to stdout (the rejected value - # is JSON-escaped in that message, so it can never smuggle in a newline or workflow command), - # which this step re-echoes so it still surfaces as an annotation. + # Re-echoes a validate-payload failure so its `::error ::…` line surfaces as an annotation. - name: Validate payload id: validate env: @@ -81,17 +57,7 @@ jobs: fi echo "$output" >>"$GITHUB_OUTPUT" - # Reconciles the service against every committed slim-services release (`planSlimUpdates`), - # not just the release that triggered this dispatch — this run is idempotent and safe to - # receive out of order. Records go only to `--output` (never stdout): each line is - # field-separated with `\x1f` (see `updateLine`), which a later step reads directly. Any - # `::warning ::…` the planner emits (an ignored tag, say) prints to this step's own stdout - # instead, so it can never be mistaken for a malformed record. - # - # `--expect-release` guards against the releases API lagging behind this very dispatch: it - # requires `-` to be a listed tag before planning runs at all, - # re-listing a bounded number of times first (`waitForExpectedRelease`). Without it, a slow - # API would make this run plan without the release that triggered it, and pass quietly. + # `--expect-release` waits for the releases API to list the release behind this dispatch. - name: Plan updates id: plan env: @@ -134,18 +100,9 @@ jobs: permission-contents: write permission-pull-requests: write - # One branch per planned item, built fresh from the base commit recorded above (the default - # branch's head at the start of this run) — never from wherever a previous loop iteration - # left HEAD. Pins exactly the planned release (`--release`, never "highest at apply time"), - # so plan and pin agree by construction: a revision minted a second later arrives with its - # own dispatch, not by racing this one. - # - # The app token (contents + pull-requests write) never reaches third-party code or disk: - # `git push` takes it only as part of an explicit URL, computed once as `push_url` - # (overridable via `PUSH_REMOTE_URL`, the seam a dry run uses to target a local bare repo - # instead), and `gh` gets it inline per call. Every `bun`/`pnpm` command below — the sync - # script, the Dockerfile generator, the formatter — runs under `env -u APP_TOKEN` so a - # compromised transitive dependency of any of them (they import `effect`) can't read it. + # One branch per planned item, rebuilt from the recorded base commit, never from a previous + # iteration's HEAD; `--release` pins the planned release, never "highest at apply time". + # `bun`/`pnpm` run under `env -u APP_TOKEN`: only `git push` and `gh` ever see the app token. - name: Apply planned updates if: steps.plan.outputs.count != '0' env: @@ -162,9 +119,8 @@ jobs: push_url="${PUSH_REMOTE_URL:-https://x-access-token:${APP_TOKEN}@github.com/${GITHUB_REPOSITORY}.git}" manual_prefix="bun .github/scripts/sync-artifacts-catalog.ts --service ${SERVICE} --release" - # Read the records on fd 3, not stdin: every command the loop body runs (`bun`, `gh`, - # `git`) otherwise shares stdin with the `read`, and any of them consuming a byte of it - # would swallow the rest of the file's records. + # Reads records on fd 3, not stdin, so commands the loop runs (`bun`, `gh`, `git`) never + # consume bytes meant for `read`. while IFS=$'\x1f' read -r -u 3 kind branch title release from; do if [ -z "$kind" ] || [ -z "$branch" ] || [ -z "$title" ] || [ -z "$release" ] || [ -z "$from" ]; then echo "::error ::Malformed plan-updates line: kind='${kind}' branch='${branch}' title='${title}' release='${release}' from='${from}'." @@ -200,10 +156,8 @@ jobs: else action="pins" fi - # Names the release this PR actually pins as the subject; the dispatch's triggering - # release (RELEASE_VERSION) can differ from it (a hotfix dispatch commonly also - # yields an unrelated upgrade on the same line), so it's only mentioned, not implied - # to be what changed. + # Names the release this PR actually pins, not necessarily RELEASE_VERSION: postgres + # can plan a hotfix on one line and an unrelated upgrade on another in one dispatch. body="$(printf 'This %s %s from %s to %s.\n\nhttps://github.com/supabase/slim-services/releases/tag/%s-%s\n\nPlanned after supabase/slim-services published %s. This branch is rewritten from %s whenever a newer relevant release arrives.\n' \ "$action" "$SERVICE" "$from" "$release" "$SERVICE" "$release" "$RELEASE_VERSION" "$DEFAULT_BRANCH")" diff --git a/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts b/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts index 73080b1628..f0e554551c 100644 --- a/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts +++ b/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts @@ -3,9 +3,9 @@ import { describe, expect, test, vi } from "vitest"; import { renderDockerfile } from "./render-service-dockerfile.ts"; // A minimal fixture catalog covering every slim-capable alias, so `renderDockerfile`'s -// "every alias needs exactly one line" check is satisfied by default; each test only mutates -// what it's exercising. `vi.mock` factories are hoisted above every other top-level statement, -// so the fixture pins are built inline here rather than imported from an outer module. +// one-line-per-alias check is satisfied by default; each test only mutates what it's +// exercising. `vi.mock` factories are hoisted above every other top-level statement, so the +// fixture pins are built inline here rather than imported from an outer module. vi.mock("@supabase/stack/internal/artifacts", () => { const nativePin = { archive: "a".repeat(64), manifest: "b".repeat(64) }; const natives = { "darwin-arm64": nativePin, "linux-amd64": nativePin, "linux-arm64": nativePin }; diff --git a/apps/cli/scripts/render-service-dockerfile.unit.test.ts b/apps/cli/scripts/render-service-dockerfile.unit.test.ts index 0c4f9036d9..029a235af8 100644 --- a/apps/cli/scripts/render-service-dockerfile.unit.test.ts +++ b/apps/cli/scripts/render-service-dockerfile.unit.test.ts @@ -21,8 +21,7 @@ describe("renderDockerfile against the real catalog and Dockerfile", () => { }); // The Go tree still `go:embed`s its own copy for a dependency that hasn't been removed yet; - // this is the single place that keeps the two copies in sync (folded from the former - // dockerfile-go-sync.unit.test.ts), until apps/cli-go is deleted. + // this is the single place that keeps the two copies in sync until apps/cli-go is deleted. test("the Go tree's embedded Dockerfile is a byte copy of the TS-owned one", () => { const goDockerfile = readFileSync(GO_DOCKERFILE_PATH, "utf8"); expect(goDockerfile).toBe(currentTsDockerfile); diff --git a/apps/cli/src/commands/start/services/vector.service.ts b/apps/cli/src/commands/start/services/vector.service.ts index ec49dab78b..c568499a4a 100644 --- a/apps/cli/src/commands/start/services/vector.service.ts +++ b/apps/cli/src/commands/start/services/vector.service.ts @@ -222,14 +222,9 @@ const VECTOR_HEALTHCHECK = { } as const; /** - * Creates `/etc/vector` (absent from Vector 0.58's images, both slim and upstream), writes the - * rendered `vector.yaml` via a `cat <<'EOF'` heredoc, waits on Logflare's `/health` (sinks would - * otherwise start too early), then `exec`s Vector so it stays PID 1. A TERM trap covers the wait - * so `docker stop` does not burn 10s if Logflare is still down; `-T 2` bounds each probe so a hung - * health endpoint can't defer the trap. Slim Vector ships BusyBox wget, so the wait uses - * `-q --spider` instead of GNU's `--no-verbose --tries`. Both images run as root, so `mkdir -p` - * needs no separate ownership handling, and `/var/lib/vector` (the `docker_logs` source's - * checkpoint `data_dir`) already exists in both. + * Vector 0.58's images (slim and upstream) have no `/etc/vector`, so the script creates it. The + * TERM trap keeps `docker stop` fast while Logflare's `/health` is still down, and `-T 2` bounds + * each probe so a hung endpoint can't defer the trap. */ export function buildVectorEntrypointScript( vectorYaml: string, diff --git a/apps/cli/src/shared/services/services.shared.unit.test.ts b/apps/cli/src/shared/services/services.shared.unit.test.ts index ff9e180f5c..c5399f0dd9 100644 --- a/apps/cli/src/shared/services/services.shared.unit.test.ts +++ b/apps/cli/src/shared/services/services.shared.unit.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, test } from "@effect/vitest"; import { Effect, Redacted } from "effect"; import { FetchHttpClient } from "effect/unstable/http"; -import { vi } from "vitest"; +import { afterEach, beforeEach, vi } from "vitest"; import serviceImagesDockerfile from "./Dockerfile" with { type: "text" }; import { dockerfileServiceImageRaw } from "./dockerfile-images.ts"; import { @@ -15,37 +15,35 @@ import { renderServicesWarning, } from "./services.shared.ts"; -// Only `auth` is pinned in this fixture catalog, at the current Dockerfile-independent version -// `v2.197.0-r0`, with a realistic (non-placeholder) fixture digest built to the real -// `ArtifactPin`/`NativePin` shape from `@supabase/stack/internal/artifacts`. Every other service -// is deliberately absent, so `toSlimImage` falls through to the upstream image for them — the -// permanent state for a non-slim-capable alias (kong, `pg14`, the job images): the Dockerfile's -// slim-capable lines are generated from the catalog now, so they never disagree with it. -// `vi.mock` factories are hoisted above every other top-level statement, so the fixture is -// inlined rather than referencing an outer const. -vi.mock("@supabase/stack/internal/artifacts", () => { - const digest = "260e94edb8d402555791146fcf70b8e90efdc6a81877a04e5aa26f0f416a5dd7"; - const nativePin = { archive: digest, manifest: digest }; - return { - catalogPins: () => [ - { - service: "auth", - sourceService: "auth", - pin: { - upstreamVersion: "v2.197.0", - revision: 0, - image: `ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:${digest}`, - natives: { - "darwin-arm64": nativePin, - "linux-amd64": nativePin, - "linux-arm64": nativePin, - }, - }, - }, - ], - }; +// `catalogPins` defaults to an auth-only fixture; the real-catalog tests swap in the original. +const { mockCatalogPins } = vi.hoisted(() => ({ mockCatalogPins: vi.fn() })); + +vi.mock("@supabase/stack/internal/artifacts", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, catalogPins: mockCatalogPins }; }); +const FIXTURE_DIGEST = "260e94edb8d402555791146fcf70b8e90efdc6a81877a04e5aa26f0f416a5dd7"; +const FIXTURE_NATIVE_PIN = { archive: FIXTURE_DIGEST, manifest: FIXTURE_DIGEST }; +const FIXTURE_CATALOG_PINS = [ + { + service: "auth", + sourceService: "auth", + pin: { + upstreamVersion: "v2.197.0", + revision: 0, + image: `ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:${FIXTURE_DIGEST}`, + natives: { + "darwin-arm64": FIXTURE_NATIVE_PIN, + "linux-amd64": FIXTURE_NATIVE_PIN, + "linux-arm64": FIXTURE_NATIVE_PIN, + }, + }, + }, +]; + +mockCatalogPins.mockImplementation(() => FIXTURE_CATALOG_PINS); + const ACCESS_TOKEN = Redacted.make(`sbp_${"a".repeat(40)}`); const PROJECT_REF = "abcdefghijklmnopqrst"; @@ -122,6 +120,78 @@ describe("services shared", () => { expect(postgresImageForDbMajorVersion(14)).toBe(pg14); }); + describe("against the real slim-services catalog", () => { + let actualCatalogPins: (typeof import("@supabase/stack/internal/artifacts"))["catalogPins"]; + + beforeEach(async () => { + actualCatalogPins = ( + await vi.importActual( + "@supabase/stack/internal/artifacts", + ) + ).catalogPins; + mockCatalogPins.mockImplementation(actualCatalogPins); + }); + + afterEach(() => { + mockCatalogPins.mockImplementation(() => FIXTURE_CATALOG_PINS); + }); + + test("lists slim images with versions derived from the catalog's default pins", () => { + const expectedNames = [ + "postgres", + "auth", + "postgrest", + "realtime", + "storage", + "edge-runtime", + "studio", + "pgmeta", + "analytics", + "pooler", + ]; + const rows = listLocalServiceVersions({ slim: true }); + + expect(rows.map((row) => row.name)).toEqual( + expectedNames.map((service) => `ghcr.io/supabase/cli/${service}`), + ); + + for (const row of rows) { + const service = row.name.replace("ghcr.io/supabase/cli/", ""); + const defaultPin = actualCatalogPins().find( + (entry) => entry.sourceService === service && entry.isDefault, + ); + expect(defaultPin).toBeDefined(); + expect(row.local).toBe(defaultPin?.pin.upstreamVersion); + expect(row.remote).toBe(""); + } + }); + + test("slim-translates a serviceVersions override to a non-default catalog pin", () => { + const nonDefaultPostgresPin = actualCatalogPins().find( + (entry) => entry.sourceService === "postgres" && !entry.isDefault, + ); + if (nonDefaultPostgresPin === undefined) { + throw new Error("Expected the catalog to carry a non-default postgres pin."); + } + const version = nonDefaultPostgresPin.pin.upstreamVersion; + + // The Dockerfile's `pg` stage pins the default line, not this one — establishing that the + // override below actually changes the resolved version instead of matching it by accident. + expect(dockerfileServiceImageRaw("pg").split(":").at(-1)).not.toBe(version); + + expect( + listLocalServiceVersions({ + slim: true, + serviceVersions: { postgres: version }, + }), + ).toContainEqual({ + name: "ghcr.io/supabase/cli/postgres", + local: version, + remote: "", + }); + }); + }); + test("slim-translates a version override that matches a catalog pin", () => { expect( listLocalServiceVersions({ slim: true, serviceVersions: { auth: "v2.197.0" } }), diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index 440e3a5284..503114e438 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -4,12 +4,7 @@ import { Config, ConfigProvider, Effect, Option } from "effect"; const SLIM_IMAGES_ENV = "SUPABASE_USE_SLIM_IMAGES"; const SLIM_IMAGE_PREFIX = "ghcr.io/supabase/cli/"; -/** - * Maps embedded-Dockerfile aliases onto the slim service catalog. Aliases with - * no slim build (kong, `pg14`, the `differ`/`migra`/`pgprove` job images) are - * absent and keep their docker.io reference. OrioleDB tags are excluded in - * `slimCatalogPin`. - */ +/** Dockerfile alias to slim service; kong, `pg14`, and the job images have no slim build. */ const SLIM_SERVICE_BY_ALIAS = { pg: "postgres", pg15: "postgres", @@ -46,19 +41,12 @@ const V_PREFIXED_SERVICES: ReadonlySet = new Set([ "pooler", ]); -/** - * Reads the ambient slim-image flag for callers without an explicit project value: always the - * process environment, never the active `ConfigProvider`. That lookup cannot fail. - */ +/** The slim-image flag from the process environment, never the active `ConfigProvider`. */ export const slimImagesEnabled = Effect.suspend(() => Config.option(Config.string(SLIM_IMAGES_ENV)).parse(ConfigProvider.fromEnv()), ).pipe(Effect.map(Option.exists((value) => value === "true" || value === "1")), Effect.orDie); -/** - * Catalog-normalized slim tag under `ghcr.io/supabase/cli/`. The - * published slim catalog uses a `v` prefix for application services while - * postgres, studio, and vector retain their unprefixed tags. - */ +/** Catalog tag: `v`-prefixed for application services; postgres, studio, and vector stay bare. */ function slimTagForService(service: SlimServiceName, rawTag: string): string { const tag = rawTag.trim(); if (V_PREFIXED_SERVICES.has(service)) { @@ -100,12 +88,7 @@ export function slimCatalogPin(alias: string, image: string): SlimCatalogPin | u return { service, version: slimTagForService(service, tag) }; } -/** - * Looks up the pinned catalog image (with its published `@sha256` digest) for - * `service` whose `upstreamVersion` equals `version`. Reads the same catalog - * `apps/cli`'s stack-independent clients use, keyed by the slim-services - * `sourceService` name (which matches this module's `SlimServiceName`). - */ +/** Keyed by the slim-services `sourceService`, which matches `SlimServiceName`. */ function catalogImageFor(service: SlimServiceName, upstreamVersion: string): string | undefined { for (const entry of catalogPins()) { if (entry.sourceService === service && entry.pin.upstreamVersion === upstreamVersion) { @@ -116,17 +99,9 @@ function catalogImageFor(service: SlimServiceName, upstreamVersion: string): str } /** - * Resolves the catalog's pinned slim image (repository, release version and - * digest) whose `upstreamVersion` normalizes to `image`'s tag for `alias`. - * This owns tag normalization (`v`-prefixing, `tagPrefix`, vector's `-alpine` - * strip) via {@link slimCatalogPin}, so pins that differ only in prefix - * between the two registries (`supavisor`, `logflare`) still match. Returns `undefined` whenever - * no catalog pin matches `alias` and `image`'s tag — callers then keep the upstream (non-slim) - * image instead of guessing a slim tag. That covers more than "no slim build": an alias with no - * slim build at all (kong, `pg14`, the one-shot job images); an excluded tag on an alias that - * does have one (an OrioleDB `pg` tag, which {@link slimCatalogPin} always excludes); and a tag - * the catalog simply doesn't pin (an upstream version the catalog hasn't caught up to yet, or a - * hosted-project override from `supabase link` that doesn't match the pinned upstream version). + * Resolves the catalog's pinned slim image matching `alias`'s normalized tag (via + * {@link slimCatalogPin}), or `undefined` when no catalog pin matches — callers then keep the + * upstream image instead of guessing a slim tag. */ export function toSlimImage(alias: string, image: string): string | undefined { const pin = slimCatalogPin(alias, image); diff --git a/apps/cli/src/shared/services/slim-images.unit.test.ts b/apps/cli/src/shared/services/slim-images.unit.test.ts index 5064bd026a..998611c15b 100644 --- a/apps/cli/src/shared/services/slim-images.unit.test.ts +++ b/apps/cli/src/shared/services/slim-images.unit.test.ts @@ -14,14 +14,8 @@ import { usesSlimImageRuntime, } from "./slim-images.ts"; -// Only `auth` is pinned in this fixture catalog, at `v2.197.0-r0`, with a realistic -// (non-placeholder) fixture digest built to the real `ArtifactPin`/`NativePin` shape from -// `@supabase/stack/internal/artifacts`. Every other service is deliberately absent, so -// `toSlimImage` falls through to the upstream image for them — the permanent state for a -// non-slim-capable alias (kong, `pg14`, the job images): the Dockerfile's slim-capable lines -// are generated from the catalog now, so they never disagree with it. `vi.mock` factories are -// hoisted above every other top-level statement, so the fixture is inlined rather than -// referencing an outer const. +// Only `auth` is pinned, so every other alias falls through to its upstream image. `vi.mock` +// factories are hoisted above top-level statements, so the fixture is inlined. vi.mock("@supabase/stack/internal/artifacts", () => { const digest = "d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c"; const nativePin = { archive: digest, manifest: digest }; diff --git a/apps/cli/tests/helpers/slim-images.ts b/apps/cli/tests/helpers/slim-images.ts index 0bab6745fc..eb04f57183 100644 --- a/apps/cli/tests/helpers/slim-images.ts +++ b/apps/cli/tests/helpers/slim-images.ts @@ -5,14 +5,7 @@ import { slimCatalogPin } from "../../src/shared/services/slim-images.ts"; /** A regression to the docker.io fallback must fail an assertion built from this. */ export const GHCR_SLIM_IMAGE_PATTERN = /^ghcr\.io\/supabase\/cli\/.+@sha256:[0-9a-f]{64}$/; -/** - * The catalog's own pinned image for `alias`'s (docker.io) `image` — read straight from - * `catalogPins()`, independent of `toSlimImage`, so a test asserting against this actually - * exercises the catalog lookup instead of passing whether or not it resolves (a default - * Dockerfile tag is generated from the catalog, so it always matches a catalog pin). Only reuses `slimCatalogPin` for alias - * and tag normalization (`v`-prefixing), not the catalog image lookup itself. Throws when - * nothing is pinned, so a caller never silently falls back to a weaker assertion. - */ +/** The catalog's pinned image for `alias`'s `image`, read independently of `toSlimImage`. */ export function expectedPinnedImage(alias: string, image: string): string { const pin = slimCatalogPin(alias, image); if (pin === undefined) { diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index cba056ffe3..0917bfbc80 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -70,6 +70,25 @@ runtime checksum authority: **GitHub Releases, the S3 mirror, GHCR, and ECR Publ mirrors only** for both images and native archives. None of them is consulted for the expected hash — that comes only from the pin already committed to the catalog. +### Invocations + +`.github/scripts/sync-artifacts-catalog.ts` has three modes: + +- **Manual** — `--service [--upstream | --release -r]`. Refreshes one catalog + entry; see "Sync and the S3-staleness check" below. +- **`plan-updates --service --output [--format lines] [--expect-release -r]`** + — used by `slim-release-published.yml`. Runs `planSlimUpdates` (see "Hotfix and upgrade pickup" + below) against the service's committed releases and writes the resulting records only to + `--output`, never to stdout; any `::warning ::…` the planner emits goes to stdout instead, so + the workflow can read warnings there without risking mistaking one for a malformed record. + `--expect-release` guards against the releases API lagging behind the dispatch that triggered + this run (`waitForExpectedRelease`): before planning, `-` must already be a + listed tag, or this mode re-lists a bounded number of times before giving up. +- **`validate-payload --service --upstream --revision --release `** — used by the + same workflow, before anything else. Checks an untrusted dispatch payload against anchored + charsets and prints it back as `key=value` lines, so a value that fails validation never reaches + `$GITHUB_OUTPUT`. + ### Sync and the S3-staleness check `.github/scripts/sync-artifacts-catalog.ts` writes those pins, in manual mode: given a service @@ -79,35 +98,62 @@ release's `SHA256SUMS` for the archive and manifest sha256 per target, and resol digest with `regctl manifest head`. Before writing the pin, it downloads each target's S3 archive and manifest and hashes them against those same release sums. This exists because `publish-release` does not wait for the ECR/S3 mirror to finish, so a freshly committed -revision's S3 copy can briefly lag. A missing object waits, bounded (`waitForExpectedRelease`'s -retry helper, generalized); an object that exists with the wrong bytes fails the sync -immediately — that's corruption, not lag, and means "run the mirror backfill", not "the CLI is -broken". Hosts that reach GitHub are unaffected — GitHub is the primary mirror — but a host that +revision's S3 copy can briefly lag. A missing object is waited for, bounded; an object that +exists with the wrong bytes fails the sync immediately — that's corruption, not lag, and means +"run the mirror backfill", not "the CLI is broken". Hosts that reach GitHub are unaffected — GitHub is the primary mirror — but a host that can only reach S3 would otherwise fail verification with no fallback. ### Hotfix and upgrade pickup The last step of slim-services' `publish-release` sends a `repository_dispatch` (`slim-release-published`) to the CLI repo with `{service, upstream_version, revision, -release_version}`. `slim-release-published.yml` treats the dispatch as a trigger, not as the -payload to apply: it reconciles the named service against every committed (published, non-draft) -`-...-r` release it can currently see (`planSlimUpdates`), and opens or updates, per -release line the service carries: +release_version}`. The payload arrives with whatever authority holds the dispatch token, so +`slim-release-published.yml` treats it as untrusted: `validate-payload` mode checks every field +against an anchored charset before it is written to `$GITHUB_OUTPUT`, a branch name, or a PR +title, and the workflow only ever passes those fields through `env:`, never interpolating them +into a `run:` script. Release tag names come from the releases API too, so `planSlimUpdates` +re-validates every value it emits against the same patterns before it can reach a branch name or +PR title. + +The workflow treats the dispatch as a trigger, not as the payload to apply: it reconciles the +named service against every committed (published, non-draft) `-...-r` release it can +currently see (`planSlimUpdates`), and opens or updates, per release line the service carries, at +most one of: - a **hotfix**, when the pinned upstream version has a higher committed revision — branch `slim-hotfix/[-]`, title `chore(stack): pin `; - an **upgrade**, when the newest committed upstream on that line is newer than the pinned one — branch `slim-bump/[-]`, title `chore(stack): bump to `. -Both can be planned in the same run. Plan and apply run from the same checkout of the default -branch in one job, so a re-run always recomputes from the latest develop: a stale plan can never -be applied, and a superseded PR's branch is rewritten (force-pushed) in place rather than raced -by a new one — the workflow deliberately never auto-closes a superseded PR. A backlog republish -of an older upstream version naturally plans nothing. The fallback, if the push or PR step fails, -is a documented manual `bun .github/scripts/sync-artifacts-catalog.ts --service --release --r` invocation, followed by `apps/cli/scripts/render-service-dockerfile.ts` — the release -itself is already committed by then, so a failure here means "open the pull request by hand", not -"republish". +A line that upgrades skips its hotfix. Both PRs would edit the same catalog span, so once the +upgrade merged, the `slim-hotfix/[-]` PR would be left conflicting, and no later run +revisits it because the pin has moved past that upstream. The planner instead emits a +`::warning ::…` naming the skipped release and the manual `--release` invocation that pins it +alone. Different lines stay independent, so one run can still plan both kinds: postgres can +upgrade its 17 line while hotfixing its 15 line. + +A service with a single pin has a single line, which accepts any comparable newer upstream: a +Studio year rollover or a postgrest major bump moves that line forward. Only a service with +additional pins (postgres) assigns each release tag to a line by its leading version component. A +tag on no carried line, or with a version that isn't comparable, is warned about and ignored +rather than failing the run. Comparison strips a trailing `-sha-`, so two Studio builds dated +the same day compare equal and never produce an upgrade; the manual `--release` path pins such a +build. + +Plan and apply run from the same checkout of the default branch in one job, so a re-run always +recomputes from the latest develop: a stale plan can never be applied, and a superseded PR's +branch is rewritten (force-pushed) in place rather than raced by a new one — a superseded PR is +never auto-closed. A backlog republish of an older upstream version naturally plans nothing. The +fallback, if the push or PR step fails, is a documented manual `bun +.github/scripts/sync-artifacts-catalog.ts --service --release -r` invocation, followed +by `apps/cli/scripts/render-service-dockerfile.ts` — the release itself is already committed by +then, so a failure here means "open the pull request by hand", not "republish". + +The app token (contents and pull-requests write) never reaches third-party code or disk: `git +push` takes it only inside an explicit URL (`PUSH_REMOTE_URL` overrides it, so a dry run can +target a local bare repository), and each `gh` call gets it inline. The sync script, the +Dockerfile generator and the formatter all run with it unset, so a compromised transitive +dependency of any of them cannot read it. ### Registry and bucket mirrors diff --git a/packages/stack/src/services/Vector.ts b/packages/stack/src/services/Vector.ts index aeab546a93..ec830a7c99 100644 --- a/packages/stack/src/services/Vector.ts +++ b/packages/stack/src/services/Vector.ts @@ -22,9 +22,7 @@ export interface Creation extends Schema.Schema.Type {} /** * Vector has no API flag or env var, so the recipe loads this alongside the pipeline config. - * `address` is templated directly into the file content at write time (not through Vector's own - * `${VAR}` config interpolation, which 0.58 disabled by default) — this is the only stack-owned - * config Vector loads that ever varied per launch, so no interpolation flag is needed at all. + * `address` is written into the file because Vector 0.58 disables `${VAR}` interpolation. */ const apiConfigFor = (address: string | undefined): string => address === undefined @@ -32,13 +30,8 @@ const apiConfigFor = (address: string | undefined): string => : `api:\n enabled: true\n address: "${address}"\n`; /** - * Every `${VAR}` name the recipe's own `env` sets and documents for a pipeline config - * (`Config.analyticsUrl`/`apiKey`, mirrored into `LOGFLARE_URL`/`LOGFLARE_PRIVATE_ACCESS_TOKEN`). - * A caller-supplied pipeline (`Config.configPath`) may reference these the same way the recipe's - * own API config used to. `renderKnownPlaceholders` substitutes only this closed, recipe-owned - * set directly into the file at write time — never a caller's or the process's arbitrary - * environment — so a caller config keeps working on Vector 0.58 without the recipe ever passing - * `--dangerously-allow-env-var-interpolation` (which would expose every env var, not just these). + * Substitutes the recipe-owned `${VAR}` names a caller pipeline (`Config.configPath`) may use, so + * it runs without `--dangerously-allow-env-var-interpolation`, which exposes every env var. */ const renderKnownPlaceholders = ( content: string, From f47f9bc290d1640a1dfa9a1367644746de0e9109 Mon Sep 17 00:00:00 2001 From: Julien Goux Date: Wed, 30 Sep 2026 23:07:38 +0200 Subject: [PATCH 2/2] chore(ci): drop async mock setup from the listing tests and document hotfix skip consequences --- .../services/services.shared.unit.test.ts | 23 +++++++++---------- docs/adr/0026-slim-artifact-mirrors.md | 11 +++++++-- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/apps/cli/src/shared/services/services.shared.unit.test.ts b/apps/cli/src/shared/services/services.shared.unit.test.ts index c5399f0dd9..93179986c7 100644 --- a/apps/cli/src/shared/services/services.shared.unit.test.ts +++ b/apps/cli/src/shared/services/services.shared.unit.test.ts @@ -18,10 +18,16 @@ import { // `catalogPins` defaults to an auth-only fixture; the real-catalog tests swap in the original. const { mockCatalogPins } = vi.hoisted(() => ({ mockCatalogPins: vi.fn() })); -vi.mock("@supabase/stack/internal/artifacts", async (importOriginal) => { - const actual = await importOriginal(); - return { ...actual, catalogPins: mockCatalogPins }; -}); +vi.mock("@supabase/stack/internal/artifacts", (importOriginal) => + importOriginal().then((actual) => ({ + ...actual, + catalogPins: mockCatalogPins, + })), +); + +const { catalogPins: actualCatalogPins } = await vi.importActual< + typeof import("@supabase/stack/internal/artifacts") +>("@supabase/stack/internal/artifacts"); const FIXTURE_DIGEST = "260e94edb8d402555791146fcf70b8e90efdc6a81877a04e5aa26f0f416a5dd7"; const FIXTURE_NATIVE_PIN = { archive: FIXTURE_DIGEST, manifest: FIXTURE_DIGEST }; @@ -121,14 +127,7 @@ describe("services shared", () => { }); describe("against the real slim-services catalog", () => { - let actualCatalogPins: (typeof import("@supabase/stack/internal/artifacts"))["catalogPins"]; - - beforeEach(async () => { - actualCatalogPins = ( - await vi.importActual( - "@supabase/stack/internal/artifacts", - ) - ).catalogPins; + beforeEach(() => { mockCatalogPins.mockImplementation(actualCatalogPins); }); diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index 0917bfbc80..3ea6dd22ff 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -100,8 +100,9 @@ and manifest and hashes them against those same release sums. This exists becaus `publish-release` does not wait for the ECR/S3 mirror to finish, so a freshly committed revision's S3 copy can briefly lag. A missing object is waited for, bounded; an object that exists with the wrong bytes fails the sync immediately — that's corruption, not lag, and means -"run the mirror backfill", not "the CLI is broken". Hosts that reach GitHub are unaffected — GitHub is the primary mirror — but a host that -can only reach S3 would otherwise fail verification with no fallback. +"run the mirror backfill", not "the CLI is broken". Hosts that reach GitHub are unaffected — +GitHub is the primary mirror — but a host that can only reach S3 would otherwise fail +verification with no fallback. ### Hotfix and upgrade pickup @@ -132,6 +133,12 @@ revisits it because the pin has moved past that upstream. The planner instead em alone. Different lines stay independent, so one run can still plan both kinds: postgres can upgrade its 17 line while hotfixing its 15 line. +The skip lasts as long as the upgrade is available: while its PR stays open, or if it is +declined, every run skips the line's hotfix again, and only the manual `--release` invocation +pins it. A hotfix PR opened before the upgrade appeared is left as is. Merge it before the +upgrade and the upgrade PR conflicts until the next run for that service rewrites it; merge the +upgrade first and the hotfix PR is superseded and must be closed by hand. + A service with a single pin has a single line, which accepts any comparable newer upstream: a Studio year rollover or a postgrest major bump moves that line forward. Only a service with additional pins (postgres) assigns each release tag to a line by its leading version component. A