diff --git a/.github/scripts/mirror-slim-image.test.ts b/.github/scripts/mirror-slim-image.test.ts new file mode 100644 index 0000000000..d81d6bc6a0 --- /dev/null +++ b/.github/scripts/mirror-slim-image.test.ts @@ -0,0 +1,285 @@ +import { describe, expect, test } from "bun:test"; + +import { + copyImage, + copyNatives, + ensureEcrPublicRepo, + main, + nativesFromEvent, + verifyDigest, + type CommandResult, + type RunCommand, +} from "./mirror-slim-image.ts"; +import { + InvalidPayloadError, + digestReference, + nativeTagPattern, + parseNatives, + validateMirrorDispatch, +} from "./slim-mirror-payload.ts"; + +const DIGEST = `sha256:${"a".repeat(64)}`; +const OTHER = `sha256:${"b".repeat(64)}`; + +const ok = (stdout = ""): CommandResult => ({ ok: true, stdout, stderr: "" }); +const fail = (stderr = "", stdout = ""): CommandResult => ({ ok: false, stdout, stderr }); + +describe("validateMirrorDispatch", () => { + test("derives source and destination on workflow_dispatch", () => { + expect( + validateMirrorDispatch({ + eventName: "workflow_dispatch", + service: "postgrest", + version: "v16.2", + digest: DIGEST, + payloadSource: undefined, + payloadDestination: undefined, + }), + ).toEqual({ + service: "postgrest", + version: "v16.2", + digest: DIGEST, + source: "ghcr.io/supabase/cli/postgrest:v16.2", + destination: "public.ecr.aws/supabase/cli/postgrest:v16.2", + }); + }); + + test("requires payload URLs to match derived refs on repository_dispatch", () => { + expect(() => + validateMirrorDispatch({ + eventName: "repository_dispatch", + service: "postgrest", + version: "v16.2", + digest: DIGEST, + payloadSource: "ghcr.io/evil/cli/postgrest:v16.2", + payloadDestination: "public.ecr.aws/supabase/cli/postgrest:v16.2", + }), + ).toThrow(InvalidPayloadError); + }); + + test("rejects a digest that is not sha256", () => { + expect(() => + validateMirrorDispatch({ + eventName: "workflow_dispatch", + service: "postgrest", + version: "v16.2", + digest: "sha256:nope", + payloadSource: undefined, + payloadDestination: undefined, + }), + ).toThrow(InvalidPayloadError); + }); +}); + +describe("parseNatives", () => { + test("accepts an empty or missing list", () => { + expect(parseNatives(undefined, "v16.2")).toEqual([]); + expect(parseNatives(null, "v16.2")).toEqual([]); + expect(parseNatives([], "v16.2")).toEqual([]); + }); + + test("rejects a platform image tag", () => { + expect(() => parseNatives([{ tag: "v16.2-linux-arm64", digest: DIGEST }], "v16.2")).toThrow( + InvalidPayloadError, + ); + }); + + test("keeps a matching native tag", () => { + expect(parseNatives([{ tag: "v16.2-native-linux-arm64", digest: DIGEST }], "v16.2")).toEqual([ + { tag: "v16.2-native-linux-arm64", digest: DIGEST }, + ]); + expect(nativeTagPattern("v16.2").test("v16.2-native-darwin-arm64")).toBe(true); + }); +}); + +describe("verifyDigest", () => { + test("accepts a matching head", async () => { + const run: RunCommand = async () => ok(`${DIGEST}\n`); + await verifyDigest({ + reference: "ghcr.io/supabase/cli/postgrest:v16.2", + digest: DIGEST, + run, + log: () => undefined, + }); + }); + + test("rejects a mismatch", async () => { + const run: RunCommand = async () => ok(`${OTHER}\n`); + await expect( + verifyDigest({ + reference: "ghcr.io/supabase/cli/postgrest:v16.2", + digest: DIGEST, + run, + log: () => undefined, + }), + ).rejects.toThrow(InvalidPayloadError); + }); +}); + +describe("ensureEcrPublicRepo", () => { + test("no-ops when the repository exists", async () => { + const run: RunCommand = async () => ok(); + const logs: string[] = []; + await ensureEcrPublicRepo({ + service: "postgrest", + run, + log: (message) => logs.push(message), + }); + expect(logs).toEqual(["ECR Public repository cli/postgrest exists"]); + }); + + test("creates a missing repository", async () => { + const run: RunCommand = async (argv) => + argv.includes("describe-repositories") ? fail("not found") : ok(); + const logs: string[] = []; + await ensureEcrPublicRepo({ + service: "postgrest", + run, + log: (message) => logs.push(message), + }); + expect(logs).toEqual(["created ECR Public repository cli/postgrest"]); + }); + + test("treats a create race as success", async () => { + const run: RunCommand = async () => fail("RepositoryAlreadyExistsException"); + const logs: string[] = []; + await ensureEcrPublicRepo({ + service: "auth", + run, + log: (message) => logs.push(message), + }); + expect(logs).toEqual(["ECR Public repository cli/auth was created concurrently"]); + }); + + test("fails when create is denied", async () => { + const run: RunCommand = async () => fail("AccessDenied"); + await expect( + ensureEcrPublicRepo({ service: "postgrest", run, log: () => undefined }), + ).rejects.toThrow(/CreateRepository/); + }); +}); + +describe("copyImage", () => { + test("copies by digest with referrers", async () => { + const calls: string[][] = []; + const run: RunCommand = async (argv) => { + calls.push([...argv]); + return ok(); + }; + await copyImage({ + source: "ghcr.io/supabase/cli/postgrest:v16.2", + destination: "public.ecr.aws/supabase/cli/postgrest:v16.2", + digest: DIGEST, + run, + }); + expect(calls).toEqual([ + [ + "regctl", + "image", + "copy", + "--referrers", + "--digest-tags", + digestReference("ghcr.io/supabase/cli/postgrest:v16.2", DIGEST), + "public.ecr.aws/supabase/cli/postgrest:v16.2", + ], + ]); + }); +}); + +describe("copyNatives", () => { + test("copies each matching source and continues after a missing tag", async () => { + const calls: string[][] = []; + const run: RunCommand = async (argv) => { + calls.push([...argv]); + if (argv[1] === "manifest" && String(argv[3]).includes("linux-amd64")) return fail("missing"); + if (argv[1] === "manifest") return ok(`${DIGEST}\n`); + return ok(); + }; + const warnings: string[] = []; + const failed = await copyNatives({ + service: "postgrest", + natives: [ + { tag: "v16.2-native-linux-arm64", digest: DIGEST }, + { tag: "v16.2-native-linux-amd64", digest: DIGEST }, + ], + run, + log: (message) => { + warnings.push(message); + }, + }); + expect(failed).toBe(1); + expect(calls).toEqual([ + ["regctl", "manifest", "head", "ghcr.io/supabase/cli/postgrest:v16.2-native-linux-arm64"], + [ + "regctl", + "image", + "copy", + `ghcr.io/supabase/cli/postgrest@${DIGEST}`, + "public.ecr.aws/supabase/cli/postgrest:v16.2-native-linux-arm64", + ], + ["regctl", "manifest", "head", "ghcr.io/supabase/cli/postgrest:v16.2-native-linux-amd64"], + ]); + expect(warnings.some((line) => line.includes("linux-amd64 is missing"))).toBe(true); + }); + + test("skips a digest mismatch without copying", async () => { + const calls: string[][] = []; + const run: RunCommand = async (argv) => { + calls.push([...argv]); + return argv[1] === "manifest" ? ok(`${OTHER}\n`) : fail("should not copy"); + }; + const failed = await copyNatives({ + service: "postgrest", + natives: [{ tag: "v16.2-native-linux-arm64", digest: DIGEST }], + run, + log: () => undefined, + }); + expect(failed).toBe(1); + expect(calls).toEqual([ + ["regctl", "manifest", "head", "ghcr.io/supabase/cli/postgrest:v16.2-native-linux-arm64"], + ]); + }); +}); + +describe("main", () => { + test("validate writes GitHub outputs", async () => { + const fields: Record = {}; + const code = await main(["validate"], { + env: { + EVENT_NAME: "workflow_dispatch", + SERVICE: "postgrest", + VERSION: "v16.2", + DIGEST, + }, + run: async () => fail("unused"), + writeOutput: (next) => { + Object.assign(fields, next); + }, + }); + expect(code).toBe(0); + expect(fields["destination"]).toBe("public.ecr.aws/supabase/cli/postgrest:v16.2"); + }); + + test("copy-natives no-ops without a dispatch payload", async () => { + const logs: string[] = []; + const code = await main(["copy-natives"], { + env: { EVENT_NAME: "workflow_dispatch", SERVICE: "postgrest", VERSION: "v16.2" }, + run: async () => fail("unused"), + log: (message) => logs.push(message), + }); + expect(code).toBe(0); + expect(logs).toEqual(["no native artifacts in payload"]); + }); + + test("copy-natives reads natives from the dispatch event", async () => { + const natives = await nativesFromEvent( + { EVENT_NAME: "repository_dispatch", GITHUB_EVENT_PATH: "event.json" }, + "v16.2", + async () => + JSON.stringify({ + client_payload: { natives: [{ tag: "v16.2-native-linux-arm64", digest: DIGEST }] }, + }), + ); + expect(natives).toEqual([{ tag: "v16.2-native-linux-arm64", digest: DIGEST }]); + }); +}); diff --git a/.github/scripts/mirror-slim-image.ts b/.github/scripts/mirror-slim-image.ts new file mode 100644 index 0000000000..b957e0abfa --- /dev/null +++ b/.github/scripts/mirror-slim-image.ts @@ -0,0 +1,294 @@ +/** + * GHCR → ECR Public copy for slim images and optional native OCI tags. + * Registry logins stay in the workflow; this script is the replayable logic. + * + * bun .github/scripts/mirror-slim-image.ts validate + * bun .github/scripts/mirror-slim-image.ts verify-digest + * bun .github/scripts/mirror-slim-image.ts ensure-repo + * bun .github/scripts/mirror-slim-image.ts copy-image + * bun .github/scripts/mirror-slim-image.ts copy-natives + */ + +import { appendFileSync } from "node:fs"; + +import { + DEST_REGISTRY, + InvalidPayloadError, + SOURCE_REGISTRY, + digestReference, + parseNatives, + validateMirrorDispatch, + type NativeArtifact, +} from "./slim-mirror-payload.ts"; + +export type CommandResult = { + readonly ok: boolean; + readonly stdout: string; + readonly stderr: string; +}; + +export type RunCommand = (argv: ReadonlyArray) => Promise; + +export type MirrorIo = { + readonly env: NodeJS.Dict; + readonly run: RunCommand; + readonly log?: (message: string) => void; + readonly readText?: (path: string) => Promise; + readonly writeOutput?: (fields: Readonly>) => void; +}; + +const usage = `Usage: + bun .github/scripts/mirror-slim-image.ts validate + bun .github/scripts/mirror-slim-image.ts verify-digest + bun .github/scripts/mirror-slim-image.ts ensure-repo + bun .github/scripts/mirror-slim-image.ts copy-image + bun .github/scripts/mirror-slim-image.ts copy-natives`; + +const envValue = (env: NodeJS.Dict, key: string): string => env[key] ?? ""; + +const requireEnv = (env: NodeJS.Dict, key: string): string => { + const value = envValue(env, key).trim(); + if (value === "") throw new InvalidPayloadError(`missing required environment variable: ${key}`); + return value; +}; + +const writeGithubOutput = ( + env: NodeJS.Dict, + fields: Readonly>, + writeOutput?: MirrorIo["writeOutput"], +): void => { + if (writeOutput !== undefined) { + writeOutput(fields); + return; + } + const body = `${Object.entries(fields) + .map(([key, value]) => `${key}=${value}`) + .join("\n")}\n`; + const path = env["GITHUB_OUTPUT"]; + if (path !== undefined && path.trim() !== "") appendFileSync(path, body); + else process.stdout.write(body); +}; + +export const verifyDigest = async (options: { + readonly reference: string; + readonly digest: string; + readonly run: RunCommand; + readonly log?: (message: string) => void; +}): Promise => { + const log = options.log ?? console.log; + const head = await options.run(["regctl", "manifest", "head", options.reference]); + const live = head.stdout.trim(); + if (!head.ok || live !== options.digest) { + const resolved = head.ok ? live : live || "missing"; + const detail = !head.ok && head.stderr.trim() !== "" ? `: ${head.stderr.trim()}` : ""; + throw new InvalidPayloadError( + `${options.reference} resolves to ${resolved}, expected ${options.digest}${detail}`, + ); + } + log(`${options.reference} resolves to ${options.digest}`); +}; + +export const ensureEcrPublicRepo = async (options: { + readonly service: string; + readonly run: RunCommand; + readonly log?: (message: string) => void; +}): Promise => { + const log = options.log ?? console.log; + const repoName = `cli/${options.service}`; + const described = await options.run([ + "aws", + "ecr-public", + "describe-repositories", + "--repository-names", + repoName, + "--region", + "us-east-1", + ]); + if (described.ok) { + log(`ECR Public repository ${repoName} exists`); + return; + } + const created = await options.run([ + "aws", + "ecr-public", + "create-repository", + "--repository-name", + repoName, + "--region", + "us-east-1", + ]); + if (created.ok) { + log(`created ECR Public repository ${repoName}`); + return; + } + const detail = `${created.stdout}\n${created.stderr}`; + if (detail.includes("RepositoryAlreadyExistsException")) { + log(`ECR Public repository ${repoName} was created concurrently`); + return; + } + if (detail.trim() !== "") log(detail.trim()); + throw new InvalidPayloadError( + `ECR Public repository '${repoName}' does not exist and this role cannot create it (missing ecr-public:CreateRepository). Create it once manually — aws ecr-public create-repository --repository-name '${repoName}' --region us-east-1 — then re-run this workflow.`, + ); +}; + +export const copyImage = async (options: { + readonly source: string; + readonly destination: string; + readonly digest: string; + readonly run: RunCommand; +}): Promise => { + const copied = await options.run([ + "regctl", + "image", + "copy", + "--referrers", + "--digest-tags", + digestReference(options.source, options.digest), + options.destination, + ]); + if (!copied.ok) + throw new InvalidPayloadError(copied.stderr.trim() || `copy failed: ${options.source}`); +}; + +export const copyNatives = async (options: { + readonly service: string; + readonly natives: ReadonlyArray; + readonly run: RunCommand; + readonly log?: (message: string) => void; +}): Promise => { + const log = options.log ?? console.log; + let failed = 0; + for (const { tag, digest } of options.natives) { + const source = `${SOURCE_REGISTRY}/${options.service}:${tag}`; + const destination = `${DEST_REGISTRY}/${options.service}:${tag}`; + const head = await options.run(["regctl", "manifest", "head", source]); + const sourceDigest = head.stdout.trim(); + if (!head.ok) { + log(`::warning::native source ${source} is missing`); + if (head.stderr.trim() !== "") log(head.stderr.trim()); + failed += 1; + continue; + } + if (sourceDigest !== digest) { + log(`::warning::native source ${source} resolves to ${sourceDigest}, expected ${digest}`); + failed += 1; + continue; + } + const copied = await options.run([ + "regctl", + "image", + "copy", + digestReference(source, digest), + destination, + ]); + if (!copied.ok) { + log(`::warning::native copy failed: ${source} -> ${destination}`); + if (copied.stderr.trim() !== "") log(copied.stderr.trim()); + failed += 1; + continue; + } + log(`mirrored ${destination}@${digest}`); + } + return failed; +}; + +export const nativesFromEvent = async ( + env: NodeJS.Dict, + version: string, + readText: (path: string) => Promise, +): Promise> => { + if (envValue(env, "EVENT_NAME") !== "repository_dispatch") return []; + const path = envValue(env, "GITHUB_EVENT_PATH").trim(); + if (path === "") + throw new InvalidPayloadError("GITHUB_EVENT_PATH is required for repository_dispatch"); + const event: unknown = JSON.parse(await readText(path)); + const payload = + typeof event === "object" && event !== null && "client_payload" in event + ? (event as { client_payload?: { natives?: unknown } }).client_payload + : undefined; + return parseNatives(payload?.natives, version); +}; + +const defaultReadText = (path: string): Promise => Bun.file(path).text(); + +const defaultSpawn: RunCommand = async (argv) => { + const proc = Bun.spawn([...argv], { stdout: "pipe", stderr: "pipe" }); + const [stdout, stderr, exit] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + return { ok: exit === 0, stdout, stderr }; +}; + +export const main = async (argv: ReadonlyArray, io: MirrorIo): Promise => { + const command = argv[0]; + const log = io.log ?? console.log; + const readText = io.readText ?? defaultReadText; + if (command === undefined || command === "-h" || command === "--help") { + log(usage); + return command === undefined ? 2 : 0; + } + if (command === "validate") { + const refs = validateMirrorDispatch({ + eventName: envValue(io.env, "EVENT_NAME"), + service: envValue(io.env, "SERVICE"), + version: envValue(io.env, "VERSION"), + digest: envValue(io.env, "DIGEST"), + payloadSource: io.env["PAYLOAD_SOURCE"], + payloadDestination: io.env["PAYLOAD_DESTINATION"], + }); + writeGithubOutput(io.env, refs, io.writeOutput); + return 0; + } + if (command === "verify-digest") { + await verifyDigest({ + reference: requireEnv(io.env, "REFERENCE"), + digest: requireEnv(io.env, "DIGEST"), + run: io.run, + log, + }); + return 0; + } + if (command === "ensure-repo") { + await ensureEcrPublicRepo({ + service: requireEnv(io.env, "SERVICE"), + run: io.run, + log, + }); + return 0; + } + if (command === "copy-image") { + await copyImage({ + source: requireEnv(io.env, "SOURCE"), + destination: requireEnv(io.env, "DESTINATION"), + digest: requireEnv(io.env, "DIGEST"), + run: io.run, + }); + return 0; + } + if (command === "copy-natives") { + const service = requireEnv(io.env, "SERVICE"); + const version = requireEnv(io.env, "VERSION"); + const natives = await nativesFromEvent(io.env, version, readText); + if (natives.length === 0) { + log("no native artifacts in payload"); + return 0; + } + return await copyNatives({ service, natives, run: io.run, log }); + } + log(usage); + return 2; +}; + +if (import.meta.main) { + try { + const code = await main(process.argv.slice(2), { env: process.env, run: defaultSpawn }); + process.exit(code); + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + console.error(`::error::${message}`); + process.exit(1); + } +} diff --git a/.github/scripts/slim-mirror-payload.ts b/.github/scripts/slim-mirror-payload.ts new file mode 100644 index 0000000000..a14a4e59e6 --- /dev/null +++ b/.github/scripts/slim-mirror-payload.ts @@ -0,0 +1,108 @@ +/** + * Untrusted `mirror-slim-image` dispatch fields. Source and destination URLs + * are derived here; payload strings are only accepted when they match. + */ + +export const SERVICE_PATTERN = /^[a-z][a-z0-9-]*$/; +export const VERSION_PATTERN = /^[A-Za-z0-9._-]+$/; +export const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; + +export const SOURCE_REGISTRY = "ghcr.io/supabase/cli"; +export const DEST_REGISTRY = "public.ecr.aws/supabase/cli"; +export const NATIVE_TARGETS = ["linux-arm64", "linux-amd64", "darwin-arm64"] as const; + +export class InvalidPayloadError extends Error {} + +export const imageSource = (service: string, version: string): string => + `${SOURCE_REGISTRY}/${service}:${version}`; + +export const imageDestination = (service: string, version: string): string => + `${DEST_REGISTRY}/${service}:${version}`; + +export const digestReference = (tagged: string, digest: string): string => { + const colon = tagged.lastIndexOf(":"); + if (colon <= 0) throw new InvalidPayloadError(`invalid image reference: ${tagged}`); + return `${tagged.slice(0, colon)}@${digest}`; +}; + +export const escapeRegExp = (value: string): string => + value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +export const nativeTagPattern = (version: string): RegExp => + new RegExp(`^${escapeRegExp(version)}-native-(${NATIVE_TARGETS.join("|")})$`); + +/** `service` / `version` / `digest` are the image fields. Extra keys such as `natives[]` are ignored. */ +export const validatePayload = (input: { + readonly service: string; + readonly version: string; + readonly digest: string; +}): void => { + if (!SERVICE_PATTERN.test(input.service)) + throw new InvalidPayloadError(`invalid service name: '${input.service}'`); + if (!VERSION_PATTERN.test(input.version)) + throw new InvalidPayloadError(`invalid version: '${input.version}'`); + if (!DIGEST_PATTERN.test(input.digest)) + throw new InvalidPayloadError(`invalid digest: '${input.digest}'`); +}; + +export type MirrorRefs = { + readonly service: string; + readonly version: string; + readonly digest: string; + readonly source: string; + readonly destination: string; +}; + +export const validateMirrorDispatch = (input: { + readonly eventName: string; + readonly service: string; + readonly version: string; + readonly digest: string; + readonly payloadSource: string | undefined; + readonly payloadDestination: string | undefined; +}): MirrorRefs => { + validatePayload(input); + const source = imageSource(input.service, input.version); + const destination = imageDestination(input.service, input.version); + if (input.eventName === "repository_dispatch") { + if (input.payloadSource !== source) + throw new InvalidPayloadError( + `payload source '${input.payloadSource}' does not match derived '${source}'`, + ); + if (input.payloadDestination !== destination) + throw new InvalidPayloadError( + `payload destination '${input.payloadDestination}' does not match derived '${destination}'`, + ); + } + return { + service: input.service, + version: input.version, + digest: input.digest, + source, + destination, + }; +}; + +export type NativeArtifact = { + readonly tag: string; + readonly digest: string; +}; + +export const parseNatives = (raw: unknown, version: string): ReadonlyArray => { + if (raw === undefined || raw === null) return []; + if (!Array.isArray(raw)) throw new InvalidPayloadError("natives must be a JSON array"); + const tagRe = nativeTagPattern(version); + const rows: NativeArtifact[] = []; + for (const item of raw) { + if (typeof item !== "object" || item === null || !("tag" in item) || !("digest" in item)) + throw new InvalidPayloadError(`invalid native entry: ${JSON.stringify(item)}`); + const tag = (item as { tag: unknown }).tag; + const digest = (item as { digest: unknown }).digest; + if (typeof tag !== "string" || !tagRe.test(tag)) + throw new InvalidPayloadError(`invalid native tag: ${String(tag)}`); + if (typeof digest !== "string" || !DIGEST_PATTERN.test(digest)) + throw new InvalidPayloadError(`invalid native digest: ${String(digest)}`); + rows.push({ tag, digest }); + } + return rows; +}; diff --git a/.github/scripts/sync-workload-catalog.ts b/.github/scripts/sync-workload-catalog.ts index cb00c23bb7..dc04440e08 100644 --- a/.github/scripts/sync-workload-catalog.ts +++ b/.github/scripts/sync-workload-catalog.ts @@ -5,26 +5,26 @@ * * Dependabot owns the Dockerfile and cannot own this table: these pins carry * image digests, which tag resolution never produces (ADR 0017). The dispatch - * payload is untrusted and revalidated here — those patterns are what keep - * `version`/`digest` inside the string literals they are written into. + * payload is untrusted and revalidated here — `slim-mirror-payload.ts` + * keeps `version`/`digest` inside the string literals they are written into. * * Run: `bun .github/scripts/sync-workload-catalog.ts` with SLIM_SERVICE, * SLIM_VERSION, SLIM_DIGEST. Exit 1 on an invalid payload; an unmodelled * service or release line is a successful no-op. */ -export const CATALOG_PATH = "packages/stack/src/model/WorkloadCatalog.ts"; +import { + InvalidPayloadError, + SOURCE_REGISTRY, + escapeRegExp, + validatePayload, +} from "./slim-mirror-payload.ts"; -/** Mirrors the payload validation in `mirror-slim-image.yml`. */ -const SERVICE_PATTERN = /^[a-z][a-z0-9-]*$/; -const VERSION_PATTERN = /^[A-Za-z0-9._-]+$/; -const DIGEST_PATTERN = /^sha256:[0-9a-f]{64}$/; +export { InvalidPayloadError, validatePayload } from "./slim-mirror-payload.ts"; -const SLIM_IMAGE_PREFIX = "ghcr.io/supabase/cli/"; +export const CATALOG_PATH = "packages/stack/src/model/WorkloadCatalog.ts"; -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} +const SLIM_IMAGE_PREFIX = `${SOURCE_REGISTRY}/`; /** Leading numeric component, `v` stripped. Only postgres carries >1 line. */ export function releaseLine(version: string): string { @@ -52,24 +52,6 @@ export type CatalogUpdatePlan = | { readonly kind: "unmodelled-service" } | { readonly kind: "unmodelled-release-line"; readonly known: ReadonlyArray }; -export class InvalidPayloadError extends Error {} - -export function validatePayload(input: { - readonly service: string; - readonly version: string; - readonly digest: string; -}): void { - if (!SERVICE_PATTERN.test(input.service)) { - throw new InvalidPayloadError(`invalid service name: '${input.service}'`); - } - if (!VERSION_PATTERN.test(input.version)) { - throw new InvalidPayloadError(`invalid version: '${input.version}'`); - } - if (!DIGEST_PATTERN.test(input.digest)) { - throw new InvalidPayloadError(`invalid digest: '${input.digest}'`); - } -} - /** `native("", "", ""` — image anchored so postgres != postgrest. */ function defaultEntryPattern(service: string): RegExp { const s = escapeRegExp(service); diff --git a/.github/workflows/github-scripts-ci.yml b/.github/workflows/github-scripts-ci.yml index b7ce2a3df5..d7cefc301d 100644 --- a/.github/workflows/github-scripts-ci.yml +++ b/.github/workflows/github-scripts-ci.yml @@ -11,6 +11,7 @@ on: - ".github/scripts/**" - ".github/workflows/ai-review.yml" - ".github/workflows/github-scripts-ci.yml" + - ".github/workflows/mirror-slim-image.yml" permissions: {} diff --git a/.github/workflows/mirror-slim-image.yml b/.github/workflows/mirror-slim-image.yml index 7f48630a15..7dce092acf 100644 --- a/.github/workflows/mirror-slim-image.yml +++ b/.github/workflows/mirror-slim-image.yml @@ -7,9 +7,11 @@ name: Mirror Slim Image # which can rewrite the index and change its digest. # # The payload arrives with whatever authority holds the dispatch token, so it is treated as -# untrusted: names are pattern-checked, source/destination are derived here rather than trusted -# from the payload, and the source must resolve to the claimed digest before anything is copied. +# untrusted: names are pattern-checked, source/destination are derived in +# `.github/scripts/mirror-slim-image.ts` rather than trusted from the payload, and the source +# must resolve to the claimed digest before anything is copied. # +# Replay locally: `bun .github/scripts/mirror-slim-image.ts --help` # Full contract: docs/design/ecr-mirror-dispatch.md in supabase/slim-services. on: @@ -41,14 +43,22 @@ concurrency: jobs: mirror: runs-on: ubuntu-latest - # The sender's poll times out after 15 minutes; fail fast instead of - # hanging past that window. - timeout-minutes: 10 + # The sender's poll times out after 15 minutes; native copies run after the + # image copy and must still finish inside that window. + timeout-minutes: 13 permissions: contents: read packages: read id-token: write steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: .bun-version + - name: Validate payload id: validate env: @@ -58,40 +68,7 @@ jobs: DIGEST: ${{ github.event.client_payload.digest || inputs.digest }} PAYLOAD_SOURCE: ${{ github.event.client_payload.source }} PAYLOAD_DESTINATION: ${{ github.event.client_payload.destination }} - run: | - set -euo pipefail - if [[ ! "$SERVICE" =~ ^[a-z][a-z0-9-]*$ ]]; then - echo "::error::invalid service name: '$SERVICE'" - exit 1 - fi - if [[ ! "$VERSION" =~ ^[A-Za-z0-9._-]+$ ]]; then - echo "::error::invalid version: '$VERSION'" - exit 1 - fi - if [[ ! "$DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "::error::invalid digest: '$DIGEST'" - exit 1 - fi - SOURCE="ghcr.io/supabase/cli/${SERVICE}:${VERSION}" - DESTINATION="public.ecr.aws/supabase/cli/${SERVICE}:${VERSION}" - # Never trust the payload's source/destination strings; require them - # to match the values derived from service + version. - if [ "$EVENT_NAME" = "repository_dispatch" ]; then - if [ "$PAYLOAD_SOURCE" != "$SOURCE" ]; then - echo "::error::payload source '$PAYLOAD_SOURCE' does not match derived '$SOURCE'" - exit 1 - fi - if [ "$PAYLOAD_DESTINATION" != "$DESTINATION" ]; then - echo "::error::payload destination '$PAYLOAD_DESTINATION' does not match derived '$DESTINATION'" - exit 1 - fi - fi - { - echo "service=$SERVICE" - echo "source=$SOURCE" - echo "destination=$DESTINATION" - echo "digest=$DIGEST" - } >> "$GITHUB_OUTPUT" + run: bun .github/scripts/mirror-slim-image.ts validate - name: Install regctl # $RUNNER_TEMP is always writable by the job user. @@ -114,15 +91,9 @@ jobs: - name: Verify source digest env: - SOURCE: ${{ steps.validate.outputs.source }} + REFERENCE: ${{ steps.validate.outputs.source }} DIGEST: ${{ steps.validate.outputs.digest }} - run: | - set -euo pipefail - SOURCE_DIGEST="$(regctl manifest head "$SOURCE")" - if [ "$SOURCE_DIGEST" != "$DIGEST" ]; then - echo "::error::source $SOURCE resolves to $SOURCE_DIGEST, expected $DIGEST" - exit 1 - fi + run: bun .github/scripts/mirror-slim-image.ts verify-digest - name: Configure aws credentials uses: aws-actions/configure-aws-credentials@cbe3b392738ccf3f987d68400dafcf4b0624a56c # v6.2.4 @@ -138,26 +109,7 @@ jobs: - name: Ensure ECR Public repository exists env: SERVICE: ${{ steps.validate.outputs.service }} - run: | - set -euo pipefail - REPO_NAME="cli/${SERVICE}" - if aws ecr-public describe-repositories \ - --repository-names "$REPO_NAME" --region us-east-1 >/dev/null 2>&1; then - echo "ECR Public repository $REPO_NAME exists" - exit 0 - fi - if CREATE_ERR="$(aws ecr-public create-repository \ - --repository-name "$REPO_NAME" --region us-east-1 2>&1 >/dev/null)"; then - echo "created ECR Public repository $REPO_NAME" - elif grep -q RepositoryAlreadyExistsException <<< "$CREATE_ERR"; then - # Concurrent run for another version of the same new service won - # the creation race; the repository exists, which is all we need. - echo "ECR Public repository $REPO_NAME was created concurrently" - else - echo "$CREATE_ERR" - echo "::error::ECR Public repository '$REPO_NAME' does not exist and this role cannot create it (missing ecr-public:CreateRepository). Create it once manually — aws ecr-public create-repository --repository-name '$REPO_NAME' --region us-east-1 — then re-run this workflow." - exit 1 - fi + run: bun .github/scripts/mirror-slim-image.ts ensure-repo - name: Mirror image env: @@ -168,20 +120,20 @@ jobs: # unconditionally with no early exit: regctl's copy is incremental, so re-dispatching # after a complete copy is a cheap no-op, and after a partial failure it finishes the # copy instead of skipping it. - run: | - set -euo pipefail - regctl image copy --referrers --digest-tags \ - "${SOURCE%:*}@${DIGEST}" "$DESTINATION" + run: bun .github/scripts/mirror-slim-image.ts copy-image - name: Verify destination digest env: - DESTINATION: ${{ steps.validate.outputs.destination }} + REFERENCE: ${{ steps.validate.outputs.destination }} DIGEST: ${{ steps.validate.outputs.digest }} - run: | - set -euo pipefail - DEST_DIGEST="$(regctl manifest head "$DESTINATION")" - if [ "$DEST_DIGEST" != "$DIGEST" ]; then - echo "::error::destination $DESTINATION resolves to $DEST_DIGEST, expected $DIGEST" - exit 1 - fi - echo "$DESTINATION resolves to $DIGEST" + run: bun .github/scripts/mirror-slim-image.ts verify-digest + + - name: Mirror native artifacts + continue-on-error: true + env: + SERVICE: ${{ steps.validate.outputs.service }} + VERSION: ${{ steps.validate.outputs.version }} + EVENT_NAME: ${{ github.event_name }} + # Native tags are copied without --referrers. Failure must not fail the + # image digest the sender polls; daily audit reports native drift. + run: bun .github/scripts/mirror-slim-image.ts copy-natives diff --git a/.github/workflows/sync-stack-workload-catalog.yml b/.github/workflows/sync-stack-workload-catalog.yml index 4ff9f7df1c..dc2d9bac03 100644 --- a/.github/workflows/sync-stack-workload-catalog.yml +++ b/.github/workflows/sync-stack-workload-catalog.yml @@ -55,7 +55,8 @@ jobs: dependency-firewall-token: ${{ secrets.DF_FIREWALL_TOKEN }} # Payload is untrusted; the script revalidates it. Passed via env, never - # interpolated into the shell. + # interpolated into the shell. Extra payload fields such as natives[] are + # ignored here; this workflow pins the image digest only. - name: Sync workload catalog env: SLIM_SERVICE: ${{ github.event.client_payload.service || inputs.service }} diff --git a/apps/cli/src/command-internal/docker-registry.ts b/apps/cli/src/command-internal/docker-registry.ts index 598aea9b27..45b8ed6618 100644 --- a/apps/cli/src/command-internal/docker-registry.ts +++ b/apps/cli/src/command-internal/docker-registry.ts @@ -3,12 +3,12 @@ * overrides the default ECR mirror; `docker.io` returns the image unchanged, and any other value * rewrites it to `/supabase/`. Callers that can retry pulls should * use {@link getRegistryImageUrlCandidates} instead, which falls back through GHCR and the - * source image. Slim images ({@link isSlimImageRef}) skip every rewrite — there's no mirror to - * redirect them to. + * source image. Slim images (`ghcr.io/supabase/cli` / `public.ecr.aws/supabase/cli`) use a + * host-prefix swap that keeps `@sha256` pins; env-hint reordering then fail-through. */ import { Config, ConfigProvider, Effect, Option } from "effect"; -import { isSlimImageRef } from "../shared/services/slim-images.ts"; +import { isSlimCatalogImage, slimImagePullCandidates } from "@supabase/stack/effect"; const INTERNAL_IMAGE_REGISTRY_ENV = "SUPABASE_INTERNAL_IMAGE_REGISTRY"; const DEFAULT_REGISTRY = "public.ecr.aws"; @@ -50,12 +50,32 @@ const registryOverride = Effect.fnUntraced(function* ( ); }); +const mergedEnv = ( + projectEnvValues?: Readonly>, +): Readonly> => + projectEnvValues === undefined ? process.env : { ...process.env, ...projectEnvValues }; + +const overrideValue = (override: Option.Option): string | undefined => { + if (Option.isNone(override) || override.value.length === 0) return undefined; + return override.value; +}; + export function getRegistryImageUrl( imageName: string, projectEnvValues?: Readonly>, ): Effect.Effect { - if (isSlimImageRef(imageName)) { - return Effect.succeed(imageName); + if (isSlimCatalogImage(imageName)) { + return registryOverride(projectEnvValues).pipe( + Effect.map((override) => { + const candidates = slimImagePullCandidates(imageName, { + env: mergedEnv(projectEnvValues), + ...(overrideValue(override) === undefined + ? {} + : { registryOverride: overrideValue(override) }), + }); + return candidates[0] ?? imageName; + }), + ); } return registryOverride(projectEnvValues).pipe( Effect.map((override) => rewriteRegistryImage(imageName, override)), @@ -66,8 +86,17 @@ export function getRegistryImageUrlCandidates( imageName: string, projectEnvValues?: Readonly>, ): Effect.Effect, Config.ConfigError> { - if (isSlimImageRef(imageName)) { - return Effect.succeed([imageName]); + if (isSlimCatalogImage(imageName)) { + return registryOverride(projectEnvValues).pipe( + Effect.map((override) => + slimImagePullCandidates(imageName, { + env: mergedEnv(projectEnvValues), + ...(overrideValue(override) === undefined + ? {} + : { registryOverride: overrideValue(override) }), + }), + ), + ); } return registryOverride(projectEnvValues).pipe( diff --git a/apps/cli/src/command-internal/docker-registry.unit.test.ts b/apps/cli/src/command-internal/docker-registry.unit.test.ts index 632948ed05..da117ce2a3 100644 --- a/apps/cli/src/command-internal/docker-registry.unit.test.ts +++ b/apps/cli/src/command-internal/docker-registry.unit.test.ts @@ -24,13 +24,28 @@ describe("getRegistryImageUrl", () => { ); const withRegistry = (value: string | undefined, fn: () => T): T => { const prev = process.env["SUPABASE_INTERNAL_IMAGE_REGISTRY"]; + const hintKeys = [ + "CLAUDE_CODE_REMOTE", + "CLAUDECODE", + "CLAUDE_CODE", + "CODEX_SANDBOX", + "CODEX_THREAD_ID", + "CODEX_CI", + "CURSOR_AGENT", + ] as const; + const hintPrev = hintKeys.map((key) => [key, process.env[key]] as const); if (value === undefined) delete process.env["SUPABASE_INTERNAL_IMAGE_REGISTRY"]; else process.env["SUPABASE_INTERNAL_IMAGE_REGISTRY"] = value; + for (const key of hintKeys) delete process.env[key]; try { return fn(); } finally { if (prev === undefined) delete process.env["SUPABASE_INTERNAL_IMAGE_REGISTRY"]; else process.env["SUPABASE_INTERNAL_IMAGE_REGISTRY"] = prev; + for (const [key, saved] of hintPrev) { + if (saved === undefined) delete process.env[key]; + else process.env[key] = saved; + } } }; @@ -122,33 +137,42 @@ describe("getRegistryImageUrl", () => { ).toBe("merged.example/supabase/pg_prove:3.36"); }); - // Published only under ghcr.io/supabase/cli; rewriting by last path segment would misroute it. + // Published under ghcr.io/supabase/cli and mirrored to public.ecr.aws/supabase/cli. const SLIM_IMAGE = "ghcr.io/supabase/cli/postgres:17.6.1.165"; + const SLIM_ECR = "public.ecr.aws/supabase/cli/postgres:17.6.1.165"; + const SLIM_PINNED = `${SLIM_IMAGE}@sha256:${"a".repeat(64)}`; + const SLIM_ECR_PINNED = `${SLIM_ECR}@sha256:${"a".repeat(64)}`; - it("leaves a slim image unrewritten, whatever the registry override says", () => { - for (const registry of [undefined, "public.ecr.aws", "docker.io", "my.mirror.example"]) { - expect(withRegistry(registry, () => resolveImage(SLIM_IMAGE))).toBe(SLIM_IMAGE); - } - expect( - withRegistry(undefined, () => - resolveImage(SLIM_IMAGE, { - SUPABASE_INTERNAL_IMAGE_REGISTRY: "my.mirror.example", - }), - ), - ).toBe(SLIM_IMAGE); + it("rewrites a slim image onto the ECR Public mirror by default", () => { + expect(withRegistry(undefined, () => resolveImage(SLIM_IMAGE))).toBe(SLIM_ECR); + expect(withRegistry(undefined, () => resolveCandidates(SLIM_IMAGE))).toEqual([ + SLIM_ECR, + SLIM_IMAGE, + ]); }); - it("plans a single pull candidate for a slim image", () => { - for (const registry of [undefined, "public.ecr.aws", "docker.io", "my.mirror.example"]) { - expect(withRegistry(registry, () => resolveCandidates(SLIM_IMAGE))).toEqual([SLIM_IMAGE]); - } + it("keeps @sha256 when swapping a slim image host", () => { + expect(withRegistry(undefined, () => resolveCandidates(SLIM_PINNED))).toEqual([ + SLIM_ECR_PINNED, + SLIM_PINNED, + ]); + }); + + it("honors a registry override for slim images", () => { + expect(withRegistry("ghcr.io", () => resolveCandidates(SLIM_IMAGE))).toEqual([SLIM_IMAGE]); + expect(withRegistry("public.ecr.aws", () => resolveCandidates(SLIM_IMAGE))).toEqual([SLIM_ECR]); + expect(withRegistry("my.mirror.example", () => resolveImage(SLIM_IMAGE))).toBe( + "my.mirror.example/supabase/cli/postgres:17.6.1.165", + ); + }); + + it("prefers GHCR for slim images when a Cursor or Codex hint is set", () => { expect( - withRegistry(undefined, () => - resolveCandidates(SLIM_IMAGE, { - SUPABASE_INTERNAL_IMAGE_REGISTRY: "my.mirror.example", - }), - ), - ).toEqual([SLIM_IMAGE]); + withRegistry(undefined, () => { + process.env["CURSOR_AGENT"] = "1"; + return resolveCandidates(SLIM_IMAGE); + }), + ).toEqual([SLIM_IMAGE, SLIM_ECR]); }); it("still rewrites the non-slim ghcr.io/supabase namespace", () => { diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index 9a27bc024e..c2ae46c0d3 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -1,5 +1,6 @@ +import { isSlimCatalogImage, SLIM_GHCR_PREFIX as SLIM_IMAGE_PREFIX } from "@supabase/stack/effect"; + const SLIM_IMAGES_ENV = "SUPABASE_USE_SLIM_IMAGES"; -const SLIM_IMAGE_PREFIX = "ghcr.io/supabase/cli/"; /** * Maps embedded-Dockerfile aliases onto the slim service catalog. Aliases with @@ -145,13 +146,13 @@ export function slimImageForCurrentPin( return toSlimImage(alias, tagged); } -/** Slim images are published only under this prefix; single home for the check. */ +/** Slim images live under GHCR or the ECR Public mirror of that namespace. */ export function isSlimImageRef(image: string): boolean { - return image.startsWith(SLIM_IMAGE_PREFIX); + return isSlimCatalogImage(image); } /** - * True when the flag is on AND `image` is a slim ghcr ref. Spec builders and + * True when the flag is on AND `image` is a slim catalog ref. Spec builders and * one-shot jobs use this so a ghcr-shaped override with the flag off stays on * the docker.io contract. */ diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md new file mode 100644 index 0000000000..89d55f8f78 --- /dev/null +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -0,0 +1,49 @@ +# 0026. Slim image and native artifact mirrors + +**Status**: proposed +**Date**: 2026-09-17 + +## Problem Statement + +Slim service images publish to `ghcr.io/supabase/cli/:`. Native archives publish to GitHub Releases. A broken build must be replaceable under the **same** upstream version string (`force=true`); bumping the tag is not an option. + +Some agent sandboxes block GitHub release-asset downloads for repositories that are not attached to the session, and they allow registry **front** hosts while blocking the **blob CDNs** those registries redirect to. A single download URL cannot serve every sandbox. + +## Decision + +Publish each slim **image** to GHCR and AWS ECR Public (`public.ecr.aws/supabase/cli/`), digest-preserving. Publish each slim **native** archive as an OCI artifact on the same two repositories under `:version-native-` (not `:version-linux-*`, which are image platform tags). GitHub Releases remain the human HTTPS copy and `--clobber` on force. + +The CLI tries an ordered candidate list. Well-known environment markers (`CLAUDE_CODE_REMOTE` / `CLAUDECODE` / `CLAUDE_CODE`, `CODEX_SANDBOX` / `CODEX_THREAD_ID` / `CODEX_CI`, `CURSOR_AGENT`) reorder that list; a failed host falls through. `SUPABASE_INTERNAL_IMAGE_REGISTRY` remains a hard override. Image rewrites are host-prefix swaps and keep `@sha256` when present ([ADR 0017](0017-simplified-managed-stack-architecture.md) digest pins). Stack catalog pulls (`RuntimeArtifacts` / `ContainerRuntime`) use the same candidates as the Docker image helper. Native OCI pulls use the registry bearer-token dance. + +`force=true` always requests the ECR copy, including when the image digest is unchanged (natives may have moved). The **image** destination digest gates `publish-release` once the dispatch token exists. Native ECR copy is best-effort and must not fail that release. Catalog sync consumes the image `service` / `version` / `digest` only; native tags ride in a separate payload field. Do not prune untagged GHCR or ECR manifests: already-shipped CLIs still pin old image digests until a catalog PR ships. Natives follow the moved tag immediately. + +ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. Do not add a public S3 bucket or a second cloud vendor for this cut. + +## Follow-up + +Claude Trusted lists `public.ecr.aws` and `ghcr.io` but 403s blob CDNs (`*.cloudfront.net`, `pkg-containers.githubusercontent.com`) and GitHub release assets for unattached repos ([claude-code#71629](https://github.com/anthropics/claude-code/issues/71629)). Later options: S3 HTTPS on `*.amazonaws.com`, or Anthropic adding blob hosts. + +## Rationale + +ECR Public mirroring already exists for other CLI images and needs no new vendor. Native OCI on those same repos reuses `regctl` and that role. Environment-aware order is required because some failures are mid-pull (manifest succeeds, blob CDN is denied), not a cheap connect miss. + +## Consequences + +- Same-version overwrite works on GHCR, GitHub Releases, and ECR Public. +- Sandboxes whose blob CDNs are not allowlisted still cannot finish an ECR or GHCR layer download until that allowlist changes or a later HTTPS host on an allowed name (for example S3 on `*.amazonaws.com`) is added. +- A GitHub native can be live while the ECR native is stale; daily mirror audit must report native drift. +- Old CLI releases keep pulling the previous image digest. + +## Alternatives considered + +1. **Google Artifact Registry / GCS** — blob host `storage.googleapis.com` is on at least one major sandbox Trusted list. Rejected for this cut: new company-wide vendor. +2. **Public S3 bucket** — HTTPS GET on `*.amazonaws.com` can work without CloudFront. Rejected for this cut: new public bucket, IAM, and security review on `PROD_AWS_ROLE`. +3. **npm packages** — allowlisted widely, but a published version cannot be replaced. +4. **Docker Hub `supabase/cli-*`** — deferred; blob CDN is also off some default allowlists, and names collide with upstream `supabase/`. +5. **Unpin slim images** — would make old CLIs see a moved tag. Rejected: conflicts with ADR 0017. + +## Related + +- [ADR 0011](0011-cli-release-and-distribution-strategy.md) — CLI binary distribution (npm + GitHub Releases) +- [ADR 0017](0017-simplified-managed-stack-architecture.md) — catalog digest pins +- slim-services `docs/design/ecr-mirror-dispatch.md` — dispatch contract diff --git a/docs/adr/README.md b/docs/adr/README.md index 083f7fb36e..b25e8af88b 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -66,6 +66,7 @@ When an ADR becomes outdated, mark it as `deprecated` or reference the supersedi | 0023 | [Config Pull Write Strategy and Scope Resolution](0023-config-pull-write-strategy-and-scope-resolution.md) | accepted | | 0024 | [Top-Level `pull` Orchestration](0024-top-level-pull-orchestration.md) | accepted | | 0025 | [Ephemeral Postgres for Schema Tooling](0025-ephemeral-postgres-for-schema-tooling.md) | proposed | +| 0026 | [Slim Image and Native Artifact Mirrors](0026-slim-artifact-mirrors.md) | proposed | ## Template diff --git a/packages/stack/src/model/SlimArtifactMirrors.ts b/packages/stack/src/model/SlimArtifactMirrors.ts new file mode 100644 index 0000000000..2fc5c63a45 --- /dev/null +++ b/packages/stack/src/model/SlimArtifactMirrors.ts @@ -0,0 +1,130 @@ +export const SLIM_GHCR_PREFIX = "ghcr.io/supabase/cli/"; +export const SLIM_ECR_PREFIX = "public.ecr.aws/supabase/cli/"; + +export type ArtifactHostHint = "default" | "claude" | "codex" | "cursor"; + +export type NativeFetchCandidate = + | { + readonly kind: "github"; + readonly downloadUrl: string; + readonly manifestUrl: string; + readonly checksumUrl: string; + } + | { + readonly kind: "oci"; + readonly registry: string; + readonly repository: string; + readonly tag: string; + }; + +const present = (value: string | undefined): boolean => value !== undefined && value.trim() !== ""; + +/** Claude / Codex / Cursor markers reorder candidates; they never drop a host. */ +export const detectArtifactHostHint = ( + env: Readonly>, +): ArtifactHostHint => { + if ( + present(env["CLAUDE_CODE_REMOTE"]) || + present(env["CLAUDECODE"]) || + present(env["CLAUDE_CODE"]) + ) + return "claude"; + if (present(env["CODEX_SANDBOX"]) || present(env["CODEX_THREAD_ID"]) || present(env["CODEX_CI"])) + return "codex"; + if (present(env["CURSOR_AGENT"])) return "cursor"; + return "default"; +}; + +export const isSlimCatalogImage = (image: string): boolean => + image.startsWith(SLIM_GHCR_PREFIX) || image.startsWith(SLIM_ECR_PREFIX); + +const slimSuffix = (image: string): string | undefined => { + if (image.startsWith(SLIM_GHCR_PREFIX)) return image.slice(SLIM_GHCR_PREFIX.length); + if (image.startsWith(SLIM_ECR_PREFIX)) return image.slice(SLIM_ECR_PREFIX.length); + return undefined; +}; + +/** Host-prefix swap that keeps `:tag` and `@sha256` when present. */ +export const rewriteSlimImageHost = (image: string, hostPrefix: string): string => { + const suffix = slimSuffix(image); + if (suffix === undefined) return image; + const prefix = hostPrefix.endsWith("/") ? hostPrefix : `${hostPrefix}/`; + return `${prefix}${suffix}`; +}; + +type SlimRegistryMapping = + | { readonly kind: "unchanged" } + | { readonly kind: "github" } + | { readonly kind: "prefix"; readonly value: string }; + +const overridePrefix = (registryOverride: string): SlimRegistryMapping => { + const registry = registryOverride.trim().toLowerCase(); + if (registry.length === 0) return { kind: "unchanged" }; + if (registry === "docker.io") return { kind: "github" }; + if (registry === "ghcr.io") return { kind: "prefix", value: SLIM_GHCR_PREFIX }; + if (registry === "public.ecr.aws") return { kind: "prefix", value: SLIM_ECR_PREFIX }; + return { kind: "prefix", value: `${registry}/supabase/cli/` }; +}; + +const mappingFor = (registryOverride: string | undefined): SlimRegistryMapping => + registryOverride === undefined ? { kind: "unchanged" } : overridePrefix(registryOverride); + +export const slimImagePullCandidates = ( + image: string, + options: { + readonly env?: Readonly>; + readonly registryOverride?: string; + } = {}, +): ReadonlyArray => { + if (!isSlimCatalogImage(image)) return [image]; + const mapped = mappingFor(options.registryOverride); + if (mapped.kind === "prefix") return [rewriteSlimImageHost(image, mapped.value)]; + if (mapped.kind === "github") return [rewriteSlimImageHost(image, SLIM_GHCR_PREFIX)]; + const ecr = rewriteSlimImageHost(image, SLIM_ECR_PREFIX); + const ghcr = rewriteSlimImageHost(image, SLIM_GHCR_PREFIX); + const hint = detectArtifactHostHint(options.env ?? {}); + const ordered = hint === "codex" || hint === "cursor" ? [ghcr, ecr] : [ecr, ghcr]; + return [...new Set(ordered)]; +}; + +const nativeOciTag = (version: string, target: string): string => `${version}-native-${target}`; + +export const nativeArtifactCandidates = ( + artifact: { + readonly service: string; + readonly version: string; + readonly target: string; + readonly downloadUrl: string; + readonly manifestUrl: string; + readonly checksumUrl: string; + }, + options: { + readonly env?: Readonly>; + readonly registryOverride?: string; + } = {}, +): ReadonlyArray => { + const repository = `supabase/cli/${artifact.service}`; + const tag = nativeOciTag(artifact.version, artifact.target); + const github: NativeFetchCandidate = { + kind: "github", + downloadUrl: artifact.downloadUrl, + manifestUrl: artifact.manifestUrl, + checksumUrl: artifact.checksumUrl, + }; + const oci = (registry: string): NativeFetchCandidate => ({ + kind: "oci", + registry, + repository, + tag, + }); + const mapped = mappingFor(options.registryOverride); + if (mapped.kind === "github") return [github]; + if (mapped.kind === "prefix") { + const host = mapped.value.replace(/\/supabase\/cli\/$/u, "").replace(/\/$/u, ""); + return [oci(host)]; + } + const hint = detectArtifactHostHint(options.env ?? {}); + const ecr = oci("public.ecr.aws"); + const ghcr = oci("ghcr.io"); + return hint === "codex" || hint === "cursor" ? [ghcr, github, ecr] : [ecr, ghcr, github]; +}; diff --git a/packages/stack/src/model/SlimArtifactMirrors.unit.test.ts b/packages/stack/src/model/SlimArtifactMirrors.unit.test.ts new file mode 100644 index 0000000000..676df021bd --- /dev/null +++ b/packages/stack/src/model/SlimArtifactMirrors.unit.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from "@effect/vitest"; +import { + detectArtifactHostHint, + isSlimCatalogImage, + nativeArtifactCandidates, + rewriteSlimImageHost, + slimImagePullCandidates, +} from "./SlimArtifactMirrors.ts"; + +const PINNED = + "ghcr.io/supabase/cli/postgres:17.6.1.168@sha256:936536bb1f97bcab0e30f58613545f8a75676c185c5eb5b86d8f8b33ca3063a1"; +const ECR_PINNED = + "public.ecr.aws/supabase/cli/postgres:17.6.1.168@sha256:936536bb1f97bcab0e30f58613545f8a75676c185c5eb5b86d8f8b33ca3063a1"; + +const githubUrls = { + service: "postgrest", + version: "v16.2", + target: "linux-arm64", + downloadUrl: + "https://github.com/supabase/slim-services/releases/download/postgrest-v16.2/postgrest-v16.2-linux-arm64.tar.zst", + manifestUrl: + "https://github.com/supabase/slim-services/releases/download/postgrest-v16.2/postgrest-v16.2-linux-arm64.manifest.json", + checksumUrl: + "https://github.com/supabase/slim-services/releases/download/postgrest-v16.2/SHA256SUMS", +}; + +describe("SlimArtifactMirrors", () => { + it("detects well-known sandbox markers", () => { + expect(detectArtifactHostHint({})).toBe("default"); + expect(detectArtifactHostHint({ CLAUDE_CODE_REMOTE: "1" })).toBe("claude"); + expect(detectArtifactHostHint({ CLAUDECODE: "1" })).toBe("claude"); + expect(detectArtifactHostHint({ CODEX_SANDBOX: "seatbelt" })).toBe("codex"); + expect(detectArtifactHostHint({ CURSOR_AGENT: "1" })).toBe("cursor"); + }); + + it("swaps slim hosts without dropping a digest pin", () => { + expect(rewriteSlimImageHost(PINNED, "public.ecr.aws/supabase/cli/")).toBe(ECR_PINNED); + expect(rewriteSlimImageHost(ECR_PINNED, "ghcr.io/supabase/cli/")).toBe(PINNED); + expect(isSlimCatalogImage(ECR_PINNED)).toBe(true); + expect(isSlimCatalogImage("ghcr.io/supabase/postgres:17.6")).toBe(false); + }); + + it("defaults slim image pulls to ECR then GHCR", () => { + expect(slimImagePullCandidates(PINNED, { env: {} })).toEqual([ECR_PINNED, PINNED]); + expect(slimImagePullCandidates(PINNED, { env: { CLAUDE_CODE_REMOTE: "1" } })).toEqual([ + ECR_PINNED, + PINNED, + ]); + }); + + it("prefers GHCR for Codex and Cursor image pulls", () => { + expect(slimImagePullCandidates(PINNED, { env: { CURSOR_AGENT: "1" } })).toEqual([ + PINNED, + ECR_PINNED, + ]); + expect(slimImagePullCandidates(PINNED, { env: { CODEX_CI: "1" } })).toEqual([ + PINNED, + ECR_PINNED, + ]); + }); + + it("treats SUPABASE_INTERNAL_IMAGE_REGISTRY as a hard override", () => { + expect( + slimImagePullCandidates(PINNED, { env: {}, registryOverride: "public.ecr.aws" }), + ).toEqual([ECR_PINNED]); + expect(slimImagePullCandidates(PINNED, { env: {}, registryOverride: "ghcr.io" })).toEqual([ + PINNED, + ]); + expect( + slimImagePullCandidates(PINNED, { env: {}, registryOverride: "my.mirror.example" }), + ).toEqual([ + "my.mirror.example/supabase/cli/postgres:17.6.1.168@sha256:936536bb1f97bcab0e30f58613545f8a75676c185c5eb5b86d8f8b33ca3063a1", + ]); + }); + + it("leaves non-slim images as a single candidate", () => { + expect(slimImagePullCandidates("supabase/pg_prove:3.36", { env: {} })).toEqual([ + "supabase/pg_prove:3.36", + ]); + }); + + it("orders native fetches ECR, GHCR, GitHub by default", () => { + expect(nativeArtifactCandidates(githubUrls, { env: {} })).toEqual([ + { + kind: "oci", + registry: "public.ecr.aws", + repository: "supabase/cli/postgrest", + tag: "v16.2-native-linux-arm64", + }, + { + kind: "oci", + registry: "ghcr.io", + repository: "supabase/cli/postgrest", + tag: "v16.2-native-linux-arm64", + }, + { + kind: "github", + downloadUrl: githubUrls.downloadUrl, + manifestUrl: githubUrls.manifestUrl, + checksumUrl: githubUrls.checksumUrl, + }, + ]); + }); + + it("prefers GHCR then GitHub for Cursor native fetches", () => { + const candidates = nativeArtifactCandidates(githubUrls, { env: { CURSOR_AGENT: "1" } }); + expect(candidates.map((candidate) => candidate.kind)).toEqual(["oci", "github", "oci"]); + expect(candidates[0]).toMatchObject({ registry: "ghcr.io" }); + }); +}); diff --git a/packages/stack/src/preparation/ArtifactStore.ts b/packages/stack/src/preparation/ArtifactStore.ts index 14c30f2b9e..03798ee356 100644 --- a/packages/stack/src/preparation/ArtifactStore.ts +++ b/packages/stack/src/preparation/ArtifactStore.ts @@ -27,7 +27,7 @@ export interface ArtifactSource { expectedSha256: string, onProgress?: (state: "downloading" | "preparing") => void, ) => Effect.Effect< - void, + string | void, StackPreparationError | ArtifactIntegrityError, FileSystem.FileSystem | Path.Path | Crypto.Crypto | ChildProcessSpawner.ChildProcessSpawner >; @@ -716,14 +716,26 @@ const makeArtifactOperation = ( const published = yield* Effect.gen(function* () { yield* ensureDirectory(fs, path, temporary, cacheRoot); const temporaryRoot = yield* ensureSafeRoot(fs, path, temporary, cacheRoot); - yield* source.materialize(request, temporary, expectedSha256, onProgress).pipe( - Effect.provideService(FileSystem.FileSystem, fs), - Effect.provideService(Path.Path, path), - Effect.provideService(Crypto.Crypto, crypto), - // The source owns the exact tar process boundary; the store only supplies the - // already-owned process service captured by its constructor. - Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childProcessSpawner), - ); + const verifiedSha256 = yield* source + .materialize(request, temporary, expectedSha256, onProgress) + .pipe( + Effect.provideService(FileSystem.FileSystem, fs), + Effect.provideService(Path.Path, path), + Effect.provideService(Crypto.Crypto, crypto), + // The source owns the exact tar process boundary; the store only supplies the + // already-owned process service captured by its constructor. + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, childProcessSpawner), + ); + const publishedSha256 = yield* typeof verifiedSha256 === "string" && verifiedSha256.length > 0 + ? validateSha256(verifiedSha256).pipe( + Effect.mapError((cause) => + metadataError("Artifact source returned an invalid SHA-256", { + key: request.key, + cause, + }), + ), + ) + : Effect.succeed(expectedSha256); const runtimePaths = yield* validateFreshRuntimePaths( fs, path, @@ -737,7 +749,7 @@ const makeArtifactOperation = ( yield* writeMetadataSync( fs, path.join(temporary, METADATA_NAME), - metadataFor(request, expectedSha256, runtimeKinds), + metadataFor(request, publishedSha256, runtimeKinds), ); if (request.executablePath !== undefined) { const executable = runtimePaths[request.executablePath]; @@ -765,17 +777,18 @@ const makeArtifactOperation = ( }); const recovered: Effect.Effect = rename.pipe(Effect.catch(recoverPublish)); - return yield* recovered; + const raced = yield* recovered; + if (raced !== undefined) return raced; + return { + key: request.key, + path: target, + sha256: publishedSha256, + requiredRuntimePaths: [...request.requiredRuntimePaths], + ...(request.executablePath === undefined ? {} : { executablePath: request.executablePath }), + outcome: "downloaded" as const, + }; }).pipe(Effect.onExit(() => cleanup(fs, temporary))); - if (published !== undefined) return published; - return { - key: request.key, - path: target, - sha256: expectedSha256, - requiredRuntimePaths: [...request.requiredRuntimePaths], - ...(request.executablePath === undefined ? {} : { executablePath: request.executablePath }), - outcome: "downloaded" as const, - }; + return published; }); export const makeArtifactStore = ( diff --git a/packages/stack/src/preparation/RuntimeArtifacts.ts b/packages/stack/src/preparation/RuntimeArtifacts.ts index eac0dee437..ffc9a3b344 100644 --- a/packages/stack/src/preparation/RuntimeArtifacts.ts +++ b/packages/stack/src/preparation/RuntimeArtifacts.ts @@ -20,6 +20,7 @@ import { } from "./ArtifactStore.ts"; import { makeSlimServicesSource } from "./SlimServicesSource.ts"; import type { ContainerEngine } from "../runtime/ContainerEngine.ts"; +import { resolveAvailableContainerImage } from "../runtime/resolve-container-image.ts"; import { resolveContainerEngine, type ContainerEngineResolverShape, @@ -157,46 +158,36 @@ export const makeRuntimeArtifactPreparer = ( ); const image = workload.selected.image; return engine.probe.pipe( - Effect.andThen(engine.inspectImage(image)), - Effect.flatMap((inspection) => - inspection.present - ? Effect.succeed({ - workloadId: workload.id, - capability: workload.capability, - version: containerVersion(image), - outcome: "cached", - image, - }) - : Effect.sync(() => report("downloading")).pipe( - Effect.andThen(engine.pullImage(image)), - Effect.mapError( - (cause) => - new ContainerPullError({ - message: `Unable to pull container image ${image}`, - workload: workload.id, - cause, - }), - ), - Effect.as({ - workloadId: workload.id, - capability: workload.capability, - version: containerVersion(image), - outcome: "pulled" as const, - image, + Effect.mapError( + (cause) => + new ContainerEngineError({ + message: cause instanceof Error ? cause.message : "Container engine operation failed", + engine: runtime.engine, + cause, + }), + ), + Effect.andThen( + resolveAvailableContainerImage(engine, image, process.env, () => + report("downloading"), + ).pipe( + Effect.mapError( + (cause) => + new ContainerPullError({ + message: `Unable to pull container image ${image}`, + workload: workload.id, + cause, }), - ), + ), + ), ), + Effect.map((resolved) => ({ + workloadId: workload.id, + capability: workload.capability, + version: containerVersion(resolved.image), + outcome: resolved.outcome, + image: resolved.image, + })), Effect.tap(() => Effect.sync(() => report("ready"))), - Effect.mapError((cause) => - cause instanceof ContainerPullError - ? cause - : new ContainerEngineError({ - message: - cause instanceof Error ? cause.message : "Container engine operation failed", - engine: runtime.engine, - cause, - }), - ), Effect.tapError((cause) => Effect.sync(() => report("failed", cause))), ); }); diff --git a/packages/stack/src/preparation/SlimNativeOci.ts b/packages/stack/src/preparation/SlimNativeOci.ts new file mode 100644 index 0000000000..a59d933718 --- /dev/null +++ b/packages/stack/src/preparation/SlimNativeOci.ts @@ -0,0 +1,164 @@ +import { Effect, Result, Schema } from "effect"; +import { HttpClient } from "effect/unstable/http"; +import type { NativeFetchCandidate } from "../model/SlimArtifactMirrors.ts"; +import { StackPreparationError } from "../public/Errors.ts"; + +export const ARCHIVE_MEDIA_TYPE = "application/vnd.supabase.slim.archive.v1.tar+zstd"; +export const MANIFEST_MEDIA_TYPE = "application/vnd.supabase.slim.manifest.v1+json"; +export const CHECKSUM_MEDIA_TYPE = "application/vnd.supabase.slim.checksum.v1"; + +const MANIFEST_ACCEPT = + "application/vnd.oci.image.manifest.v1+json, application/vnd.oci.artifact.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json"; + +interface OciLayer { + readonly mediaType: string; + readonly digest: string; + readonly annotations: Readonly>; +} + +export interface OciNativeTriplet { + readonly manifestBytes: Uint8Array; + readonly checksumText: string; + readonly archiveDigest: string; + readonly archiveUrl: string; + readonly headers: Readonly>; +} + +const tokenUrl = (registry: string, repository: string): string => { + if (registry === "ghcr.io") + return `https://ghcr.io/token?service=ghcr.io&scope=repository:${repository}:pull`; + if (registry === "public.ecr.aws") + return `https://public.ecr.aws/token/?service=public.ecr.aws&scope=repository:${repository}:pull`; + return `https://${registry}/token?service=${registry}&scope=repository:${repository}:pull`; +}; + +const parseToken = (bytes: Uint8Array): string | undefined => { + const decoded = Schema.decodeResult(Schema.fromJsonString(Schema.Unknown))( + new TextDecoder().decode(bytes), + ); + if (Result.isFailure(decoded) || typeof decoded.success !== "object" || decoded.success === null) + return undefined; + const record = decoded.success as Record; + for (const key of ["token", "access_token", "authorizationToken"] as const) { + const value = record[key]; + if (typeof value === "string" && value.length > 0) return value; + } + return undefined; +}; + +const asLayers = (manifest: unknown): ReadonlyArray => { + if (typeof manifest !== "object" || manifest === null) return []; + const record = manifest as Record; + const raw = record["layers"] ?? record["blobs"]; + if (!Array.isArray(raw)) return []; + return raw.flatMap((entry) => { + if (typeof entry !== "object" || entry === null) return []; + const layer = entry as Record; + const mediaType = typeof layer["mediaType"] === "string" ? layer["mediaType"] : ""; + const digest = typeof layer["digest"] === "string" ? layer["digest"] : ""; + if (digest.length === 0) return []; + const annotations = + typeof layer["annotations"] === "object" && layer["annotations"] !== null + ? (layer["annotations"] as Record) + : {}; + return [{ mediaType, digest, annotations }]; + }); +}; + +const titleOf = (layer: OciLayer): string => + layer.annotations["org.opencontainers.image.title"] ?? ""; + +const findLayer = ( + layers: ReadonlyArray, + match: (layer: OciLayer) => boolean, +): OciLayer | undefined => layers.find(match); + +const responseFor = (url: string, headers?: Readonly>) => + HttpClient.get(url, headers === undefined ? undefined : { headers }).pipe( + Effect.flatMap((response) => + Effect.gen(function* () { + if (response.status < 200 || response.status >= 300) + return yield* new StackPreparationError({ message: `HTTP ${response.status}` }); + return response; + }), + ), + ); + +const fetchBytes = ( + url: string, + headers?: Readonly>, +): Effect.Effect => + responseFor(url, headers).pipe( + Effect.flatMap((response) => response.arrayBuffer), + Effect.map((bytes) => new Uint8Array(bytes)), + Effect.mapError( + (cause) => new StackPreparationError({ message: `Unable to download ${url}`, cause }), + ), + ); + +const bearerHeaders = (token: string, accept?: string): Readonly> => ({ + Authorization: `Bearer ${token}`, + ...(accept === undefined ? {} : { Accept: accept }), +}); + +const ociToken = ( + registry: string, + repository: string, +): Effect.Effect => + fetchBytes(tokenUrl(registry, repository)).pipe( + Effect.flatMap((bytes) => { + const token = parseToken(bytes); + return token === undefined + ? Effect.fail(new StackPreparationError({ message: `OCI token missing from ${registry}` })) + : Effect.succeed(token); + }), + ); + +export const fetchOciNativeTriplet = ( + candidate: Extract, +): Effect.Effect => + Effect.gen(function* () { + const token = yield* ociToken(candidate.registry, candidate.repository); + const manifestUrl = `https://${candidate.registry}/v2/${candidate.repository}/manifests/${candidate.tag}`; + const manifestBytes = yield* fetchBytes(manifestUrl, bearerHeaders(token, MANIFEST_ACCEPT)); + const parsed = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))( + new TextDecoder().decode(manifestBytes), + ).pipe( + Effect.mapError( + (cause) => + new StackPreparationError({ message: "OCI native manifest is invalid JSON", cause }), + ), + ); + const layers = asLayers(parsed); + const archive = findLayer( + layers, + (layer) => layer.mediaType === ARCHIVE_MEDIA_TYPE || titleOf(layer).endsWith(".tar.zst"), + ); + const slimManifest = findLayer( + layers, + (layer) => + layer.mediaType === MANIFEST_MEDIA_TYPE || titleOf(layer).endsWith(".manifest.json"), + ); + const checksum = findLayer( + layers, + (layer) => layer.mediaType === CHECKSUM_MEDIA_TYPE || titleOf(layer).includes("SHA256SUMS"), + ); + if (archive === undefined || slimManifest === undefined || checksum === undefined) + return yield* new StackPreparationError({ + message: `OCI native triplet is incomplete for ${candidate.registry}/${candidate.repository}:${candidate.tag}`, + }); + const blob = (digest: string) => + fetchBytes( + `https://${candidate.registry}/v2/${candidate.repository}/blobs/${digest}`, + bearerHeaders(token), + ); + const slimManifestBytes = yield* blob(slimManifest.digest); + const checksumBytes = yield* blob(checksum.digest); + return { + manifestBytes: slimManifestBytes, + checksumText: new TextDecoder().decode(checksumBytes), + archiveDigest: archive.digest, + archiveUrl: `https://${candidate.registry}/v2/${candidate.repository}/blobs/${archive.digest}`, + headers: bearerHeaders(token), + }; + }); diff --git a/packages/stack/src/preparation/SlimServicesSource.ts b/packages/stack/src/preparation/SlimServicesSource.ts index 27653ddcd4..a3c9685b6f 100644 --- a/packages/stack/src/preparation/SlimServicesSource.ts +++ b/packages/stack/src/preparation/SlimServicesSource.ts @@ -1,4 +1,15 @@ -import { Effect, FileSystem, Layer, Path, Schema, Stream } from "effect"; +import { + Config, + ConfigProvider, + Effect, + FileSystem, + Layer, + Option, + Path, + Result, + Schema, + Stream, +} from "effect"; import { NodeStream } from "@effect/platform-node"; import { FetchHttpClient, @@ -11,6 +22,11 @@ import { createHash } from "node:crypto"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { ArtifactRequest, ArtifactSource } from "./ArtifactStore.ts"; import type { NativeWorkloadArtifact } from "../model/WorkloadCatalog.ts"; +import { + nativeArtifactCandidates, + type NativeFetchCandidate, +} from "../model/SlimArtifactMirrors.ts"; +import { fetchOciNativeTriplet } from "./SlimNativeOci.ts"; import { StackPreparationError } from "../public/Errors.ts"; type Fetcher = (input: string, init?: RequestInit) => Promise; @@ -88,8 +104,8 @@ const transport = (fetchRequest?: Fetcher) => ), ); -const responseFor = (url: string) => - HttpClient.get(url).pipe( +const responseFor = (url: string, headers?: Readonly>) => + HttpClient.get(url, headers === undefined ? undefined : { headers }).pipe( Effect.flatMap((response) => Effect.gen(function* () { if (response.status < 200 || response.status >= 300) @@ -143,10 +159,11 @@ const downloadToFile = ( destination: string, request: Fetcher | undefined, expectedSha256: string, + headers?: Readonly>, ): Effect.Effect => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; - const response = yield* responseFor(url); + const response = yield* responseFor(url, headers); const hash = yield* Effect.try({ try: () => createHash("sha256"), catch: (cause) => @@ -185,24 +202,95 @@ const checksumFor = (contents: string, archiveName: string): string | undefined .map((line) => line.trim().match(/^([a-f0-9]{64})\s+[* ]?(.+)$/iu)) .find((match) => match?.[2] === archiveName || match?.[2]?.endsWith(`/${archiveName}`))?.[1]; +const CANDIDATE_ENV_KEYS = [ + "SUPABASE_INTERNAL_IMAGE_REGISTRY", + "CLAUDE_CODE_REMOTE", + "CLAUDECODE", + "CLAUDE_CODE", + "CODEX_SANDBOX", + "CODEX_THREAD_ID", + "CODEX_CI", + "CURSOR_AGENT", +] as const; + +const candidateOptions = Effect.gen(function* () { + const env: Record = {}; + const provider = ConfigProvider.fromEnv(); + for (const key of CANDIDATE_ENV_KEYS) { + const value = yield* Config.option(Config.string(key)).parse(provider); + if (Option.isSome(value) && value.value.trim() !== "") env[key] = value.value; + } + const override = env["SUPABASE_INTERNAL_IMAGE_REGISTRY"]; + return { + env, + ...(override === undefined ? {} : { registryOverride: override }), + }; +}).pipe( + Effect.mapError( + (cause) => + new StackPreparationError({ + message: "Unable to read slim artifact host configuration", + cause, + }), + ), +); + +const tryCandidates = ( + candidates: ReadonlyArray, + tryOne: (candidate: NativeFetchCandidate) => Effect.Effect, +): Effect.Effect => + Effect.gen(function* () { + const failures: Array = []; + for (const candidate of candidates) { + const result = yield* Effect.result(tryOne(candidate)); + if (Result.isSuccess(result)) return result.success; + failures.push(result.failure.message); + } + return yield* new StackPreparationError({ + message: `Unable to download from all slim artifact sources: ${failures.join("; ")}`, + }); + }); + +const parseChecksum = ( + contents: string, + archiveName: string, +): Effect.Effect => { + const checksum = checksumFor(contents, archiveName); + return checksum === undefined || !/^[a-f0-9]{64}$/iu.test(checksum) + ? Effect.fail(new StackPreparationError({ message: "Slim-services checksum is missing" })) + : Effect.succeed(checksum.toLowerCase()); +}; + export const slimServicesChecksum = ( artifact: NativeWorkloadArtifact, request?: Fetcher, ): Effect.Effect => - fetchBytes(artifact.checksumUrl, request).pipe( - Effect.map((bytes) => new TextDecoder().decode(bytes)), - Effect.flatMap((contents) => { - const checksum = checksumFor(contents, `${artifact.assetName}.tar.zst`); - return checksum === undefined || !/^[a-f0-9]{64}$/iu.test(checksum) - ? Effect.fail( - new StackPreparationError({ - message: "Slim-services checksum is missing", - service: artifact.service, - version: artifact.version, - }), - ) - : Effect.succeed(checksum.toLowerCase()); - }), + candidateOptions.pipe( + Effect.flatMap((options) => + tryCandidates(nativeArtifactCandidates(artifact, options), (candidate) => { + const contents = + candidate.kind === "github" + ? fetchBytes(candidate.checksumUrl, request).pipe( + Effect.map((bytes) => new TextDecoder().decode(bytes)), + ) + : fetchOciNativeTriplet(candidate).pipe( + Effect.provide(transport(request)), + Effect.map((triplet) => triplet.checksumText), + ); + return contents.pipe( + Effect.flatMap((text) => parseChecksum(text, `${artifact.assetName}.tar.zst`)), + Effect.mapError( + (cause) => + new StackPreparationError({ + message: "Slim-services checksum is missing", + service: artifact.service, + version: artifact.version, + cause, + }), + ), + ); + }), + ), ); const unsafeArchivePath = (value: string): boolean => { @@ -295,7 +383,7 @@ export const makeSlimServicesSource = ( resolveArtifact(request).pipe( Effect.flatMap((artifact) => slimServicesChecksum(artifact, fetchRequest)), ), - materialize: (request, destination, expectedSha256, onProgress) => + materialize: (request, destination, _expectedSha256, onProgress) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -307,65 +395,104 @@ export const makeSlimServicesSource = ( ]); return yield* Effect.gen(function* () { const artifact = yield* resolveArtifact(request); - const manifestBytes = yield* fetchBytes(artifact.manifestUrl, fetchRequest); - const manifestText = new TextDecoder().decode(manifestBytes); - const manifest = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))( - manifestText, - ).pipe( - Effect.mapError( - (cause) => - new StackPreparationError({ - message: "Slim-services manifest is invalid", - cause, - }), - ), - ); - if ( - typeof manifest !== "object" || - manifest === null || - !("service" in manifest) || - !("version" in manifest) || - !("target" in manifest) || - manifest.service !== artifact.service || - manifest.version !== artifact.version || - manifest.target !== artifact.target - ) - return yield* new StackPreparationError({ - message: "Slim-services manifest does not match the catalog artifact", - service: artifact.service, - version: artifact.version, - target: artifact.target, - }); - const entrypoint = "entrypoint" in manifest ? manifest.entrypoint : undefined; - const command = "cmd" in manifest ? manifest.cmd : undefined; - if ( - (entrypoint !== undefined && - (!Array.isArray(entrypoint) || - !entrypoint.every((value) => typeof value === "string") || - entrypoint.some(unsafeManifestCommand))) || - (command !== undefined && - (!Array.isArray(command) || - !command.every((value) => typeof value === "string") || - command.some(unsafeManifestCommand))) - ) - return yield* new StackPreparationError({ - message: "Slim-services manifest command is invalid", - service: artifact.service, - version: artifact.version, - }); - yield* Effect.sync(() => onProgress?.("downloading")).pipe( - Effect.andThen( - downloadToFile(artifact.downloadUrl, compressedPath, fetchRequest, expectedSha256), - ), - Effect.mapError( - (cause) => - new StackPreparationError({ - message: "Unable to download slim-services archive", - service: artifact.service, - version: artifact.version, - cause, - }), - ), + const options = yield* candidateOptions; + const archiveName = `${artifact.assetName}.tar.zst`; + const selectedSha256 = yield* tryCandidates( + nativeArtifactCandidates(artifact, options), + (candidate) => + Effect.gen(function* () { + const fetched = + candidate.kind === "github" + ? { + manifestBytes: yield* fetchBytes(candidate.manifestUrl, fetchRequest), + checksumText: new TextDecoder().decode( + yield* fetchBytes(candidate.checksumUrl, fetchRequest), + ), + download: (sha256: string) => + downloadToFile( + candidate.downloadUrl, + compressedPath, + fetchRequest, + sha256, + ), + } + : yield* fetchOciNativeTriplet(candidate).pipe( + Effect.provide(transport(fetchRequest)), + Effect.map((triplet) => ({ + manifestBytes: triplet.manifestBytes, + checksumText: triplet.checksumText, + download: (sha256: string) => + downloadToFile( + triplet.archiveUrl, + compressedPath, + fetchRequest, + sha256, + triplet.headers, + ), + })), + ); + // Pair checksum and archive on the same candidate so a stale first-host + // digest cannot reject a later host's matching bytes. + const sha256 = yield* parseChecksum(fetched.checksumText, archiveName); + const manifestText = new TextDecoder().decode(fetched.manifestBytes); + const manifest = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))( + manifestText, + ).pipe( + Effect.mapError( + (cause) => + new StackPreparationError({ + message: "Slim-services manifest is invalid", + cause, + }), + ), + ); + if ( + typeof manifest !== "object" || + manifest === null || + !("service" in manifest) || + !("version" in manifest) || + !("target" in manifest) || + manifest.service !== artifact.service || + manifest.version !== artifact.version || + manifest.target !== artifact.target + ) + return yield* new StackPreparationError({ + message: "Slim-services manifest does not match the catalog artifact", + service: artifact.service, + version: artifact.version, + target: artifact.target, + }); + const entrypoint = "entrypoint" in manifest ? manifest.entrypoint : undefined; + const command = "cmd" in manifest ? manifest.cmd : undefined; + if ( + (entrypoint !== undefined && + (!Array.isArray(entrypoint) || + !entrypoint.every((value) => typeof value === "string") || + entrypoint.some(unsafeManifestCommand))) || + (command !== undefined && + (!Array.isArray(command) || + !command.every((value) => typeof value === "string") || + command.some(unsafeManifestCommand))) + ) + return yield* new StackPreparationError({ + message: "Slim-services manifest command is invalid", + service: artifact.service, + version: artifact.version, + }); + yield* Effect.sync(() => onProgress?.("downloading")).pipe( + Effect.andThen(fetched.download(sha256)), + Effect.mapError( + (cause) => + new StackPreparationError({ + message: "Unable to download slim-services archive", + service: artifact.service, + version: artifact.version, + cause, + }), + ), + ); + return sha256; + }), ); yield* Effect.sync(() => onProgress?.("preparing")); yield* decompressor.decompress(compressedPath, archivePath); @@ -427,6 +554,7 @@ export const makeSlimServicesSource = ( message: `Slim-services archive extraction exited with code ${exitCode}`, }); yield* validateExtractedTree(fs, path, destination); + return selectedSha256; }).pipe(Effect.ensuring(cleanup)); }), }; diff --git a/packages/stack/src/preparation/runtime-artifacts.integration.test.ts b/packages/stack/src/preparation/runtime-artifacts.integration.test.ts index 471ac1a399..e472430418 100644 --- a/packages/stack/src/preparation/runtime-artifacts.integration.test.ts +++ b/packages/stack/src/preparation/runtime-artifacts.integration.test.ts @@ -22,6 +22,7 @@ import { import { ContainerEngineError, StackPreparationError } from "../public/Errors.ts"; import { ContainerEngineProtocolError } from "../runtime/ContainerEngine.ts"; import { catalogReleaseFor } from "../model/WorkloadCatalog.ts"; +import { slimImagePullCandidates } from "../model/SlimArtifactMirrors.ts"; const databaseRelease = catalogReleaseFor("database:database"); if (databaseRelease === undefined) throw new Error("Missing default database release"); @@ -315,6 +316,7 @@ describe("runtime artifact preparation", () => { native: { store: { prepare: () => Effect.die("unused") } }, containerEngine: engine, }); + const progress: Array = []; const result = Effect.runSync( runtime.prepare( { kind: "container", engine: "docker" }, @@ -322,14 +324,22 @@ describe("runtime artifact preparation", () => { kind: "container", image: databaseRelease.containerImage, }), + (status) => { + progress.push(status.state); + }, ), ); expect(result.outcome).toBe("pulled"); + const candidates = slimImagePullCandidates(databaseRelease.containerImage, { + env: process.env, + }); expect(calls).toEqual([ "probe", - `inspect:${databaseRelease.containerImage}`, - `pull:${databaseRelease.containerImage}`, + ...candidates.map((candidate) => `inspect:${candidate}`), + `pull:${candidates[0]}`, ]); + expect(result.image).toBe(candidates[0]); + expect(progress).toEqual(["preparing", "downloading", "ready"]); }); it("uses the persisted container engine identity", () => { diff --git a/packages/stack/src/preparation/slim-services.integration.test.ts b/packages/stack/src/preparation/slim-services.integration.test.ts index 2ad2be58c6..9a27825997 100644 --- a/packages/stack/src/preparation/slim-services.integration.test.ts +++ b/packages/stack/src/preparation/slim-services.integration.test.ts @@ -110,6 +110,14 @@ type FetchLike = ( const requestUrl = (input: Parameters[0]): string => typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; +/** GitHub fixture hosts only; registry URLs 404 so fail-through reaches example.test. */ +const githubOnly = + (inner: FetchLike): FetchLike => + (input, init) => + requestUrl(input).startsWith("https://example.test/") + ? inner(input, init) + : Promise.resolve(new Response("oci unavailable", { status: 404 })); + describe("slim-services artifact source", () => { it.live("verifies checksums and extracts a manifest-matched archive using injected fetch", () => Effect.scoped( @@ -129,8 +137,8 @@ describe("slim-services artifact source", () => { ); return Promise.resolve(new Response(archive)); }; - expect(yield* slimServicesChecksum(artifact, fetcher)).toBe(expected); - const source = makeSlimServicesSource(() => artifact, fetcher); + expect(yield* slimServicesChecksum(artifact, githubOnly(fetcher))).toBe(expected); + const source = makeSlimServicesSource(() => artifact, githubOnly(fetcher)); const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-source-" }); yield* source.materialize(request, destination, expected); @@ -157,7 +165,7 @@ describe("slim-services artifact source", () => { ); return Promise.resolve(new Response(archive)); }; - const source = makeSlimServicesSource(() => artifact, fetcher); + const source = makeSlimServicesSource(() => artifact, githubOnly(fetcher)); const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-unsafe-" }); const failed = yield* source.materialize(request, destination, expected).pipe(Effect.exit); @@ -192,7 +200,7 @@ describe("slim-services artifact source", () => { }; const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-links-" }); - yield* makeSlimServicesSource(() => artifact, fetcher).materialize( + yield* makeSlimServicesSource(() => artifact, githubOnly(fetcher)).materialize( request, destination, expected, @@ -215,7 +223,7 @@ describe("slim-services artifact source", () => { ); return Promise.resolve(new Response(malformed)); }; - const failed = yield* makeSlimServicesSource(() => artifact, malformedFetcher) + const failed = yield* makeSlimServicesSource(() => artifact, githubOnly(malformedFetcher)) .materialize(request, destination, malformedDigest) .pipe(Effect.exit); expect(errorOf(failed)).toBeInstanceOf(StackPreparationError); @@ -250,7 +258,7 @@ describe("slim-services artifact source", () => { const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-link-escape-", }); - const failed = yield* makeSlimServicesSource(() => artifact, fetcher) + const failed = yield* makeSlimServicesSource(() => artifact, githubOnly(fetcher)) .materialize(request, destination, expected) .pipe(Effect.exit); expect(errorOf(failed)).toBeInstanceOf(StackPreparationError); @@ -285,7 +293,7 @@ describe("slim-services artifact source", () => { prefix: "slim-services-interrupt-", }); const fiber = yield* Effect.forkChild( - makeSlimServicesSource(() => artifact, fetcher).materialize( + makeSlimServicesSource(() => artifact, githubOnly(fetcher)).materialize( request, destination, "0".repeat(64), @@ -327,7 +335,7 @@ describe("slim-services artifact source", () => { const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-store-integrity-", }); - const source = makeSlimServicesSource(() => artifact, fetcher); + const source = makeSlimServicesSource(() => artifact, githubOnly(fetcher)); const store = yield* makeArtifactStore({ cacheRoot: root, source }); const failed = yield* store.prepare(request).pipe(Effect.exit); expect(Exit.isFailure(failed)).toBe(true); @@ -381,7 +389,7 @@ describe("slim-services artifact source", () => { const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-stream-" }); const fiber = yield* Effect.forkChild( - makeSlimServicesSource(() => artifact, fetcher).materialize( + makeSlimServicesSource(() => artifact, githubOnly(fetcher)).materialize( request, destination, "0".repeat(64), @@ -431,7 +439,7 @@ describe("slim-services artifact source", () => { const fiber = yield* Effect.forkChild( makeSlimServicesSource( () => artifact, - fetcher, + githubOnly(fetcher), systemTarBoundary, decompressor, ).materialize(request, destination, expected), @@ -466,7 +474,7 @@ describe("slim-services artifact source", () => { }; const fs = yield* FileSystem.FileSystem; const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-pax-" }); - yield* makeSlimServicesSource(() => artifact, fetcher).materialize( + yield* makeSlimServicesSource(() => artifact, githubOnly(fetcher)).materialize( request, destination, expected, @@ -483,13 +491,9 @@ describe("slim-services artifact source", () => { const crypto = yield* Crypto.Crypto; const expected = digestHex(yield* crypto.digest("SHA-256", archive)); const checksums = `${expected} demo-v1.0.0-linux-amd64.tar.zst\n`; - let checksumRequests = 0; const fetcher: FetchLike = (input) => { const url = requestUrl(input); - if (url.endsWith("SHA256SUMS")) { - checksumRequests += 1; - return Promise.resolve(new Response(checksums)); - } + if (url.endsWith("SHA256SUMS")) return Promise.resolve(new Response(checksums)); if (url.endsWith("manifest.json")) return Promise.resolve( new Response( @@ -498,16 +502,138 @@ describe("slim-services artifact source", () => { ); return Promise.resolve(new Response(archive)); }; - const source = makeSlimServicesSource(() => artifact, fetcher); + const source = makeSlimServicesSource(() => artifact, githubOnly(fetcher)); const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-store-" }); const store = yield* makeArtifactStore({ cacheRoot: root, source }); const prepared = yield* store.prepare(request); expect(prepared.outcome).toBe("downloaded"); + expect(prepared.sha256).toBe(expected); expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); expect(yield* fs.exists(`${prepared.path}/.artifact.json`)).toBe(true); - expect(checksumRequests).toBe(1); }).pipe(Effect.provide(NodeServices.layer)), ), ); + + it.live("materializes from an OCI native artifact using a registry bearer token", () => + Effect.scoped( + Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); + const crypto = yield* Crypto.Crypto; + const expected = digestHex(yield* crypto.digest("SHA-256", archive)); + const manifestJson = JSON.stringify({ + service: "demo", + version: "v1.0.0", + target: "linux-amd64", + }); + const checksums = `${expected} demo-v1.0.0-linux-amd64.tar.zst\n`; + const archiveDigest = `sha256:${"a".repeat(64)}`; + const manifestDigest = `sha256:${"b".repeat(64)}`; + const checksumDigest = `sha256:${"c".repeat(64)}`; + const fetcher: FetchLike = (input) => { + const url = requestUrl(input); + if (url.includes("/token")) + return Promise.resolve(new Response(JSON.stringify({ token: "oci-test-token" }))); + if (url.includes("/manifests/")) + return Promise.resolve( + new Response( + JSON.stringify({ + layers: [ + { + mediaType: "application/vnd.supabase.slim.archive.v1.tar+zstd", + digest: archiveDigest, + }, + { + mediaType: "application/vnd.supabase.slim.manifest.v1+json", + digest: manifestDigest, + }, + { + mediaType: "application/vnd.supabase.slim.checksum.v1", + digest: checksumDigest, + }, + ], + }), + ), + ); + if (url.endsWith(manifestDigest)) return Promise.resolve(new Response(manifestJson)); + if (url.endsWith(checksumDigest)) return Promise.resolve(new Response(checksums)); + if (url.endsWith(archiveDigest)) return Promise.resolve(new Response(archive)); + return Promise.resolve(new Response("missing", { status: 404 })); + }; + const fs = yield* FileSystem.FileSystem; + const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-oci-" }); + yield* makeSlimServicesSource(() => artifact, fetcher).materialize( + request, + destination, + expected, + ); + expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); + + it.live( + "pairs checksum and archive per candidate so a stale first host cannot pin later ones", + () => + Effect.scoped( + Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); + const crypto = yield* Crypto.Crypto; + const expected = digestHex(yield* crypto.digest("SHA-256", archive)); + const stale = "0".repeat(64); + const manifestJson = JSON.stringify({ + service: "demo", + version: "v1.0.0", + target: "linux-amd64", + }); + const staleChecksums = `${stale} demo-v1.0.0-linux-amd64.tar.zst\n`; + const githubChecksums = `${expected} demo-v1.0.0-linux-amd64.tar.zst\n`; + const archiveDigest = `sha256:${"a".repeat(64)}`; + const manifestDigest = `sha256:${"b".repeat(64)}`; + const checksumDigest = `sha256:${"c".repeat(64)}`; + const fetcher: FetchLike = (input) => { + const url = requestUrl(input); + if (url.includes("/token")) + return Promise.resolve(new Response(JSON.stringify({ token: "oci-stale-token" }))); + if (url.includes("/manifests/")) + return Promise.resolve( + new Response( + JSON.stringify({ + layers: [ + { + mediaType: "application/vnd.supabase.slim.archive.v1.tar+zstd", + digest: archiveDigest, + }, + { + mediaType: "application/vnd.supabase.slim.manifest.v1+json", + digest: manifestDigest, + }, + { + mediaType: "application/vnd.supabase.slim.checksum.v1", + digest: checksumDigest, + }, + ], + }), + ), + ); + if (url.endsWith(manifestDigest)) return Promise.resolve(new Response(manifestJson)); + if (url.endsWith(checksumDigest)) return Promise.resolve(new Response(staleChecksums)); + if (url.endsWith(archiveDigest)) return Promise.resolve(new Response(archive)); + if (url.endsWith("SHA256SUMS")) return Promise.resolve(new Response(githubChecksums)); + if (url.endsWith("manifest.json")) return Promise.resolve(new Response(manifestJson)); + if (url.startsWith("https://example.test/")) + return Promise.resolve(new Response(archive)); + return Promise.resolve(new Response("missing", { status: 404 })); + }; + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-pair-" }); + const source = makeSlimServicesSource(() => artifact, fetcher); + const store = yield* makeArtifactStore({ cacheRoot: root, source }); + const prepared = yield* store.prepare(request); + expect(prepared.outcome).toBe("downloaded"); + expect(prepared.sha256).toBe(expected); + expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); }); diff --git a/packages/stack/src/public/index.ts b/packages/stack/src/public/index.ts index dd14b58af4..83decd6bf9 100644 --- a/packages/stack/src/public/index.ts +++ b/packages/stack/src/public/index.ts @@ -66,3 +66,13 @@ export type { SchemaInitServices, SchemaInitTarget, } from "./SchemaInit.ts"; +export { + detectArtifactHostHint, + isSlimCatalogImage, + nativeArtifactCandidates, + rewriteSlimImageHost, + slimImagePullCandidates, + SLIM_ECR_PREFIX, + SLIM_GHCR_PREFIX, +} from "../model/SlimArtifactMirrors.ts"; +export type { ArtifactHostHint, NativeFetchCandidate } from "../model/SlimArtifactMirrors.ts"; diff --git a/packages/stack/src/runtime/ContainerRuntime.ts b/packages/stack/src/runtime/ContainerRuntime.ts index c108756589..efc5db81e0 100644 --- a/packages/stack/src/runtime/ContainerRuntime.ts +++ b/packages/stack/src/runtime/ContainerRuntime.ts @@ -31,6 +31,7 @@ import { type ContainerWorkloadLabels, type ContainerLogOptions, } from "./ContainerEngine.ts"; +import { resolveAvailableContainerImage } from "./resolve-container-image.ts"; import { RuntimeDriverError, type RuntimeCleanupRequest, @@ -671,13 +672,10 @@ export const makeContainerRuntime = ( yield* withEngine(key, options.engine.stopContainer(existingExact.id)); yield* withEngine(key, options.engine.removeContainer(existingExact.id)); } - yield* withEngine(key, options.engine.inspectImage(artifact.image)).pipe( - Effect.flatMap((inspected) => - inspected.present - ? Effect.void - : withEngine(key, options.engine.pullImage(artifact.image)), - ), - ); + const image = (yield* withEngine( + key, + resolveAvailableContainerImage(options.engine, artifact.image), + )).image; yield* guard; const networkResource = yield* setup.withPermit( @@ -776,7 +774,7 @@ export const makeContainerRuntime = ( for (const startup of resolution.startup ?? []) yield* runStartupProcess(key, workload, startup, { - artifact, + artifact: { ...artifact, image }, network: networkResource, resolution, ...(volumeRequest === undefined ? {} : { volumeRequest }), @@ -787,7 +785,7 @@ export const makeContainerRuntime = ( key, options.engine.createContainer({ name: nameFor(key, "workload"), - image: artifact.image, + image, labels, network: networkResource.id, mounts: resolution.mounts ?? [], diff --git a/packages/stack/src/runtime/resolve-container-image.ts b/packages/stack/src/runtime/resolve-container-image.ts new file mode 100644 index 0000000000..daf6523a98 --- /dev/null +++ b/packages/stack/src/runtime/resolve-container-image.ts @@ -0,0 +1,42 @@ +import { Effect, Result } from "effect"; +import { slimImagePullCandidates } from "../model/SlimArtifactMirrors.ts"; +import type { ContainerEngine, ContainerEngineFailure } from "./ContainerEngine.ts"; + +export interface ResolvedContainerImage { + readonly image: string; + readonly outcome: "cached" | "pulled"; +} + +/** + * Inspects every slim candidate locally, then pulls in order. Engine failures + * on inspect (daemon down) fail immediately; pull failures fall through. + */ +export const resolveAvailableContainerImage = ( + engine: Pick, + image: string, + env: Readonly> = process.env, + onPull?: () => void, +): Effect.Effect => + Effect.gen(function* () { + const override = env["SUPABASE_INTERNAL_IMAGE_REGISTRY"]; + const candidates = slimImagePullCandidates(image, { + env, + ...(override === undefined || override.trim() === "" ? {} : { registryOverride: override }), + }); + for (const candidate of candidates) { + const inspected = yield* engine.inspectImage(candidate); + if (inspected.present) return { image: candidate, outcome: "cached" }; + } + yield* Effect.sync(() => onPull?.()); + let lastError: ContainerEngineFailure | undefined; + for (const candidate of candidates) { + const pulled = yield* Effect.result(engine.pullImage(candidate)); + if (Result.isSuccess(pulled)) return { image: candidate, outcome: "pulled" }; + lastError = pulled.failure; + } + if (lastError === undefined) + return yield* engine + .inspectImage(image) + .pipe(Effect.as({ image, outcome: "cached" as const })); + return yield* lastError; + }); diff --git a/packages/stack/src/supervisor/handles.integration.test.ts b/packages/stack/src/supervisor/handles.integration.test.ts index fe573a421f..3d20930c29 100644 --- a/packages/stack/src/supervisor/handles.integration.test.ts +++ b/packages/stack/src/supervisor/handles.integration.test.ts @@ -51,6 +51,7 @@ import { makeControlClient, startControlServer } from "../control/ControlServer. import { resolveStackPaths } from "../state/Paths.ts"; import { STACK_RPC_RELEASE, type StackRpcHandlers } from "../control/StackRpc.ts"; import { catalogReleaseFor } from "../model/WorkloadCatalog.ts"; +import { slimImagePullCandidates } from "../model/SlimArtifactMirrors.ts"; import { StackOwnershipConflictError, StackPreparationError, @@ -369,7 +370,10 @@ describe("managed stack handles", { timeout: 30_000 }, () => { }).pipe(Effect.provideService(ContainerEngineResolver, resolver)); const prepared = yield* stack.prepare({ capabilities: ["database"] }); expect(prepared.capabilities).toHaveLength(1); - expect(calls).toEqual(["podman:probe", `podman:inspect:${databaseRelease.containerImage}`]); + const candidates = slimImagePullCandidates(databaseRelease.containerImage, { + env: process.env, + }); + expect(calls).toEqual(["podman:probe", `podman:inspect:${candidates[0]}`]); expect(dockerCalls).toEqual([]); }), ),