diff --git a/.github/scripts/sync-artifacts-catalog.test.ts b/.github/scripts/sync-artifacts-catalog.test.ts index ebc4d8962e..9334d37af5 100644 --- a/.github/scripts/sync-artifacts-catalog.test.ts +++ b/.github/scripts/sync-artifacts-catalog.test.ts @@ -189,6 +189,24 @@ const postgresPlannerFixture = `const workloadCatalog = { }; `; +/** `postgresPlannerFixture` plus a resolved OrioleDB line beside stock 17. */ +const postgresOrioleFixture = postgresPlannerFixture.replace( + ` natives: {}, + }, + }, + ),`, + ` natives: {}, + }, + "17.11.0.002-orioledb": { + upstreamVersion: "17.11.0.002-orioledb", + revision: 0, + image: "ghcr.io/supabase/cli/postgres:17.11.0.002-orioledb-r0@sha256:5555555555555555555555555555555555555555555555555555555555555", + natives: {}, + }, + }, + ),`, +); + describe("validateSlimReleasePublishedPayload", () => { test("rejects a newline injected into upstream_version", () => { expect(() => @@ -502,6 +520,69 @@ describe("planSlimUpdates", () => { ]); }); + test("postgres: the first OrioleDB release adds its own line and never moves stock 17", () => { + const { updates, warnings } = planSlimUpdates(postgresPlannerFixture, "postgres", [ + "postgres-17.11.0.002-orioledb-r0", + "postgres-17.11.0.002-orioledb-r1", + "postgres-17.4.1.030-orioledb", + ]); + + expect(warnings).toEqual([]); + expect(updates).toEqual([ + { + kind: "add", + line: "17-orioledb", + branch: "slim-bump/postgres-17-orioledb", + title: "chore(stack): add postgres 17.11.0.002-orioledb-r1", + toUpstream: "17.11.0.002-orioledb", + toRelease: "17.11.0.002-orioledb-r1", + }, + ]); + }); + + test("postgres: OrioleDB and stock 17 hotfix and upgrade independently", () => { + const { updates, warnings } = planSlimUpdates(postgresOrioleFixture, "postgres", [ + "postgres-17.6.1.168-r2", + "postgres-17.11.0.002-orioledb-r1", + "postgres-17.12.0.001-orioledb-r0", + ]); + + expect(updates).toEqual([ + { + kind: "hotfix", + line: "17", + branch: "slim-hotfix/postgres-17", + title: "chore(stack): pin postgres 17.6.1.168-r2", + fromRelease: "17.6.1.168-r1", + toUpstream: "17.6.1.168", + toRelease: "17.6.1.168-r2", + }, + { + kind: "upgrade", + line: "17-orioledb", + branch: "slim-bump/postgres-17-orioledb", + title: "chore(stack): bump postgres to 17.12.0.001-orioledb-r0", + fromRelease: "17.11.0.002-orioledb-r0", + toUpstream: "17.12.0.001-orioledb", + toRelease: "17.12.0.001-orioledb-r0", + }, + ]); + expect(warnings).toEqual([ + "::warning ::postgres 17.11.0.002-orioledb-r1 hotfix skipped because this line is upgrading to 17.12.0.001-orioledb-r0; run --service postgres --release 17.11.0.002-orioledb-r1 to pin the hotfix alone.", + ]); + }); + + test("postgres: an OrioleDB release for a major the catalog does not carry is ignored and warned about", () => { + const { updates, warnings } = planSlimUpdates(postgresPlannerFixture, "postgres", [ + "postgres-18.0.0.001-orioledb-r0", + ]); + + expect(updates).toEqual([]); + expect(warnings).toEqual([ + "::warning ::postgres 18.0.0.001-orioledb is not on a release line packages/stack/src/Artifacts.ts carries for it; ignoring postgres-18.0.0.001-orioledb-r0.", + ]); + }); + test("branch and title carry no - suffix for a service with a single line", () => { const { updates } = planSlimUpdates(plannerFixture, "storage", ["storage-v1.74.0-r0"]); @@ -1062,6 +1143,64 @@ describe("refreshCatalogPin", () => { }); }); +describe("refreshCatalogPin adds the OrioleDB line", () => { + const orioleIo = (revision: number, seed: string): RevisionIo => { + const release = `17.11.0.002-orioledb-r${revision}`; + const digests = nativeDigests(seed); + return { + listReleaseTags: async () => + Array.from({ length: revision + 1 }, (_, n) => `postgres-17.11.0.002-orioledb-r${n}`), + fetchChecksums: async () => checksumsFor("postgres", release, digests), + imageDigest: async () => digest(seed), + s3Sha256: matchingS3("postgres", release, digests), + fetchManifest: manifestWithUpstreamImage("supabase/postgres:17.11.0.002-orioledb"), + fetchProvenance: unusedFetchProvenance, + }; + }; + + test("inserts the first OrioleDB pin beside stock 17 and 15, then hotfixes it after formatting", async () => { + const added = await refreshCatalogPin({ + catalog: fixture, + service: "postgres", + release: "17.11.0.002-orioledb-r0", + io: orioleIo(0, "oriole-0"), + }); + + expect(added.update).toEqual({ + service: "postgres", + version: "17.11.0.002-orioledb", + revision: 0, + target: "additional", + }); + expect(added.source).toContain(postgres17Image); + expect(added.source).toContain('"15.14.1.168": { upstreamVersion: "15.14.1.168"'); + expect(added.source).toContain( + '"17.11.0.002-orioledb": { upstreamVersion: "17.11.0.002-orioledb"', + ); + + const formatted = await formatWithOxfmt(added.source); + const hotfixed = await refreshCatalogPin({ + catalog: formatted, + service: "postgres", + release: "17.11.0.002-orioledb-r1", + io: orioleIo(1, "oriole-1"), + }); + + expect(hotfixed.update).toEqual({ + service: "postgres", + version: "17.11.0.002-orioledb", + revision: 1, + previousVersion: "17.11.0.002-orioledb", + target: "additional", + }); + expect(hotfixed.source).toContain(postgres17Image); + expect(hotfixed.source).toContain( + `"ghcr.io/supabase/cli/postgres:17.11.0.002-orioledb-r1@${digest("oriole-1")}"`, + ); + expect(hotfixed.source).not.toContain(digest("oriole-0")); + }); +}); + describe("resolveRevisionPin waits for the S3 mirror", () => { test("waits while one S3 object is missing, then resolves once it appears", async () => { const digests = nativeDigests("s3-wait"); @@ -1367,4 +1506,35 @@ describe("against the real catalog", () => { expect(refreshed.source).toContain(second["darwin-arm64"].manifest); expect(refreshed.source).toContain(`upstreamImage: "supabase/gotrue:${pinnedVersion}"`); }); + + test("the OrioleDB line and stock 17 update independently", async () => { + const catalog = await Bun.file(CATALOG_PATH).text(); + expect( + planSlimUpdates(catalog, "postgres", [ + "postgres-17.11.0.002-r0", + "postgres-17.11.0.003-r0", + "postgres-17.11.0.002-orioledb-r0", + "postgres-17.11.0.002-orioledb-r1", + ]).updates, + ).toEqual([ + { + kind: "upgrade", + line: "17", + branch: "slim-bump/postgres-17", + title: "chore(stack): bump postgres to 17.11.0.003-r0", + fromRelease: "17.11.0.002-r0", + toUpstream: "17.11.0.003", + toRelease: "17.11.0.003-r0", + }, + { + kind: "hotfix", + line: "17-orioledb", + branch: "slim-hotfix/postgres-17-orioledb", + title: "chore(stack): pin postgres 17.11.0.002-orioledb-r1", + fromRelease: "17.11.0.002-orioledb-r0", + toUpstream: "17.11.0.002-orioledb", + toRelease: "17.11.0.002-orioledb-r1", + }, + ]); + }); }); diff --git a/.github/scripts/sync-artifacts-catalog.ts b/.github/scripts/sync-artifacts-catalog.ts index e568e07ab4..cd2d9ddfa9 100644 --- a/.github/scripts/sync-artifacts-catalog.ts +++ b/.github/scripts/sync-artifacts-catalog.ts @@ -98,11 +98,46 @@ function workflowCommand(kind: "error" | "warning", message: string): string { return `::${kind} ::${encoded}`; } -/** Leading numeric component, `v` stripped. Only postgres carries more than one line. */ -function releaseLine(version: string): string { - const withoutPrefix = version.replace(/^[vV]/, ""); +/** + * Upstream suffixes that put a release on its own engine-variant line beside the stock line of + * the same major, e.g. postgres `17.11.0.002-orioledb` on line `17-orioledb`. + */ +const LINE_VARIANTS: Readonly>> = { + postgres: ["-orioledb"], +}; + +function lineVariant(service: string, version: string): string | undefined { + return LINE_VARIANTS[service]?.find((suffix) => version.endsWith(suffix)); +} + +/** `version` without its engine-variant suffix, so versions on one variant line compare. */ +function withoutVariant(service: string, version: string): string { + const variant = lineVariant(service, version); + return variant === undefined ? version : version.slice(0, -variant.length); +} + +/** + * Leading numeric component, `v` stripped, plus any engine-variant suffix. Only postgres carries + * more than one line. + */ +function releaseLine(service: string, version: string): string { + const withoutPrefix = withoutVariant(service, version).replace(/^[vV]/, ""); const separator = withoutPrefix.indexOf("."); - return separator === -1 ? withoutPrefix : withoutPrefix.slice(0, separator); + const major = separator === -1 ? withoutPrefix : withoutPrefix.slice(0, separator); + return `${major}${lineVariant(service, version) ?? ""}`; +} + +/** Whether `service` assigns release tags to lines rather than accepting any newer upstream. */ +function hasReleaseLines(service: string, additional: ReadonlyArray): boolean { + return additional.length > 0 || LINE_VARIANTS[service] !== undefined; +} + +/** + * Whether an uncarried `line` may be added on first publish: an engine-variant line whose stock + * major the catalog already carries. + */ +function isAddableLine(service: string, line: string, carried: (line: string) => boolean): boolean { + return lineVariant(service, line) !== undefined && carried(withoutVariant(service, line)); } /** @@ -119,8 +154,8 @@ function releaseTagPattern(service: string, upstream?: string): RegExp { /** * Strips a leading `v`/`V` and one trailing `-sha-`, then parses the remainder as dot- - * separated integers. Returns undefined when the remainder isn't `^\d+(\.\d+)*$` — a version that - * isn't comparable this way, such as an OrioleDB-style suffix. + * separated integers. Returns undefined when the remainder isn't `^\d+(\.\d+)*$`; callers strip an + * engine-variant suffix with `withoutVariant` first. */ function comparableVersion(version: string): ReadonlyArray | undefined { const stripped = version.replace(/^[vV]/, "").replace(/-sha-[0-9a-f]+$/i, ""); @@ -150,7 +185,8 @@ export interface CatalogPinUpdate { readonly service: string; readonly version: string; readonly revision: number; - readonly previousVersion: string; + /** Absent when this update adds the line. */ + readonly previousVersion?: string; readonly target: "default" | "additional"; } @@ -572,13 +608,16 @@ function collectServicePins(source: string, service: string): ServicePins | unde type SelectedEntry = | { readonly kind: "default" | "additional"; readonly version: string; readonly span: PinSpan } + /** An addable line with no pin yet; its pin is inserted after the last additional pin. */ + | { readonly kind: "new-line"; readonly version: string; readonly insertAt: number } | { readonly kind: "unmodelled-service" } | { readonly kind: "unmodelled-release-line"; readonly known: ReadonlyArray }; /** * Locates `service`'s pin expression in `source`: the `definition("", , ...)` * default pin, and, when `version` is given, whichever pin (default or additional) sits on its - * release line. With no `version`, returns the default pin unconditionally. + * release line, or where an addable line's first pin goes. With no `version`, returns the default + * pin unconditionally. */ function selectEntry(source: string, service: string, version: string | undefined): SelectedEntry { const pins = collectServicePins(source, service); @@ -589,19 +628,26 @@ function selectEntry(source: string, service: string, version: string | undefine return { kind: "default", version: defaultPin.version, span: defaultPin }; } + const line = releaseLine(service, version); const bumpsDefault = - additional.length === 0 || releaseLine(version) === releaseLine(defaultPin.version); + !hasReleaseLines(service, additional) || line === releaseLine(service, defaultPin.version); if (bumpsDefault) { return { kind: "default", version: defaultPin.version, span: defaultPin }; } - const sameLine = additional.find((pin) => releaseLine(pin.version) === releaseLine(version)); - if (sameLine === undefined) { - return { - kind: "unmodelled-release-line", - known: [defaultPin.version, ...additional.map((pin) => pin.version)], - }; + const sameLine = additional.find((pin) => releaseLine(service, pin.version) === line); + if (sameLine !== undefined) { + return { kind: "additional", version: sameLine.version, span: sameLine }; + } + const lastAdditional = additional.at(-1); + const carried = (candidate: string) => + [defaultPin, ...additional].some((pin) => releaseLine(service, pin.version) === candidate); + if (lastAdditional !== undefined && isAddableLine(service, line, carried)) { + return { kind: "new-line", version, insertAt: lastAdditional.end }; } - return { kind: "additional", version: sameLine.version, span: sameLine }; + return { + kind: "unmodelled-release-line", + known: [defaultPin.version, ...additional.map((pin) => pin.version)], + }; } /** The `{service, upstream_version, revision, release_version}` payload a `slim-release-published` dispatch carries. */ @@ -691,18 +737,18 @@ function writePin( } /** - * One hotfix or upgrade a `slim-release-published` run should apply, on one of `service`'s - * release lines. `line` is the leading numeric component (`releaseLine`), present only when the - * service carries more than one line (only postgres does today) — it's already folded into - * `branch`, so a caller never needs to consume it separately. + * One hotfix, upgrade, or line addition a `slim-release-published` run should apply, on one of + * `service`'s release lines. `line` is the `releaseLine` key, present only when the service + * carries more than one line (only postgres does today) — it's already folded into `branch`, so a + * caller never needs to consume it separately. */ export interface SlimUpdate { - readonly kind: "hotfix" | "upgrade"; + readonly kind: "hotfix" | "upgrade" | "add"; readonly line?: string; readonly branch: string; readonly title: string; - /** The release version pinned before this update, e.g. `v2.195.0-r0`. */ - readonly fromRelease: string; + /** The release version pinned before this update, e.g. `v2.195.0-r0`; absent for `add`. */ + readonly fromRelease?: string; readonly toUpstream: string; /** The release version this update pins, e.g. `v2.195.0-r1`. */ readonly toRelease: string; @@ -716,11 +762,11 @@ export interface SlimUpdate { * catalog itself (`fromRelease`'s upstream), not only to values freshly parsed from a tag. */ function buildUpdate( - kind: "hotfix" | "upgrade", + kind: "hotfix" | "upgrade" | "add", service: string, line: string, hasLines: boolean, - pinned: { readonly upstream: string; readonly revision: number }, + pinned: { readonly upstream: string; readonly revision: number } | undefined, toUpstream: string, toRevision: number, ): SlimUpdate { @@ -734,21 +780,22 @@ function buildUpdate( if (!RELEASE_VERSION_PATTERN.test(toRelease)) { throw new InvalidPayloadError(`invalid release version: ${JSON.stringify(toRelease)}`); } - const fromRelease = `${pinned.upstream}-r${pinned.revision}`; const suffix = hasLines ? `-${line}` : ""; const branch = kind === "hotfix" ? `slim-hotfix/${service}${suffix}` : `slim-bump/${service}${suffix}`; const title = kind === "hotfix" ? `chore(stack): pin ${service} ${toRelease}` - : `chore(stack): bump ${service} to ${toRelease}`; + : kind === "add" + ? `chore(stack): add ${service} ${toRelease}` + : `chore(stack): bump ${service} to ${toRelease}`; return { kind, line: hasLines ? line : undefined, branch, title, - fromRelease, + ...(pinned === undefined ? {} : { fromRelease: `${pinned.upstream}-r${pinned.revision}` }), toUpstream, toRelease, }; @@ -787,11 +834,13 @@ export function planSlimUpdates( } const allPins = [pins.defaultPin, ...pins.additional]; - const hasLines = pins.additional.length > 0; + const hasLines = hasReleaseLines(service, pins.additional); + const compareOnLine = (a: string, b: string) => + compareVersions(withoutVariant(service, a), withoutVariant(service, b)); const pinnedByLine = new Map(); for (const span of allPins) { - const line = hasLines ? releaseLine(span.version) : SINGLE_LINE_KEY; + const line = hasLines ? releaseLine(service, span.version) : SINGLE_LINE_KEY; const text = catalog.slice(span.start, span.end); const revisionMatch = PIN_REVISION.exec(text); if (revisionMatch === null) { @@ -801,6 +850,7 @@ export function planSlimUpdates( } pinnedByLine.set(line, { upstream: span.version, revision: Number(revisionMatch[1]) }); } + const carried = (line: string) => pinnedByLine.has(line); const tagPattern = releaseTagPattern(service); const candidatesByLine = new Map>(); @@ -811,8 +861,8 @@ export function planSlimUpdates( const revision = Number(match[2]); let line: string; if (hasLines) { - line = releaseLine(upstream); - if (!pinnedByLine.has(line)) { + line = releaseLine(service, upstream); + if (!carried(line) && !isAddableLine(service, line, carried)) { warnings.push( workflowCommand( "warning", @@ -829,16 +879,11 @@ export function planSlimUpdates( candidatesByLine.set(line, list); } - const updates: SlimUpdate[] = []; - for (const [line, pinned] of pinnedByLine) { - const candidates = candidatesByLine.get(line) ?? []; - - const sameUpstreamRevisions = candidates - .filter((candidate) => candidate.upstream === pinned.upstream) - .map((candidate) => candidate.revision); - const hotfixRevision = - sameUpstreamRevisions.length > 0 ? Math.max(...sameUpstreamRevisions) : undefined; - + /** The newest comparable upstream among `candidates` and its highest committed revision. */ + const newest = ( + candidates: ReadonlyArray<{ readonly upstream: string; readonly revision: number }>, + isComparable: (upstream: string) => boolean, + ) => { const highestRevisionByUpstream = new Map(); for (const candidate of candidates) { const current = highestRevisionByUpstream.get(candidate.upstream); @@ -846,11 +891,9 @@ export function planSlimUpdates( highestRevisionByUpstream.set(candidate.upstream, candidate.revision); } } - - let bestUpstream: string | undefined; - for (const upstream of highestRevisionByUpstream.keys()) { - const comparedToPinned = compareVersions(upstream, pinned.upstream); - if (comparedToPinned === undefined) { + let best: { upstream: string; revision: number } | undefined; + for (const [upstream, revision] of highestRevisionByUpstream) { + if (!isComparable(upstream)) { warnings.push( workflowCommand( "warning", @@ -859,20 +902,38 @@ export function planSlimUpdates( ); continue; } - if (bestUpstream === undefined || (compareVersions(upstream, bestUpstream) ?? 0) > 0) { - bestUpstream = upstream; + if (best === undefined || (compareOnLine(upstream, best.upstream) ?? 0) > 0) { + best = { upstream, revision }; } } + return best; + }; + + const updates: SlimUpdate[] = []; + for (const [line, pinned] of pinnedByLine) { + const candidates = candidatesByLine.get(line) ?? []; + + const sameUpstreamRevisions = candidates + .filter((candidate) => candidate.upstream === pinned.upstream) + .map((candidate) => candidate.revision); + const hotfixRevision = + sameUpstreamRevisions.length > 0 ? Math.max(...sameUpstreamRevisions) : undefined; - if (bestUpstream !== undefined && compareVersions(bestUpstream, pinned.upstream) === 1) { - const revision = highestRevisionByUpstream.get(bestUpstream) as number; - updates.push(buildUpdate("upgrade", service, line, hasLines, pinned, bestUpstream, revision)); + const best = newest( + candidates, + (upstream) => compareOnLine(upstream, pinned.upstream) !== undefined, + ); + + if (best !== undefined && compareOnLine(best.upstream, pinned.upstream) === 1) { + updates.push( + buildUpdate("upgrade", service, line, hasLines, pinned, best.upstream, best.revision), + ); if (hotfixRevision !== undefined && hotfixRevision > pinned.revision) { const skippedRelease = `${pinned.upstream}-r${hotfixRevision}`; warnings.push( workflowCommand( "warning", - `${service} ${skippedRelease} hotfix skipped because this line is upgrading to ${bestUpstream}-r${revision}; run --service ${service} --release ${skippedRelease} to pin the hotfix alone.`, + `${service} ${skippedRelease} hotfix skipped because this line is upgrading to ${best.upstream}-r${best.revision}; run --service ${service} --release ${skippedRelease} to pin the hotfix alone.`, ), ); } @@ -883,6 +944,19 @@ export function planSlimUpdates( } } + const newLines = [...candidatesByLine.keys()].filter((line) => !carried(line)).sort(); + for (const line of newLines) { + const best = newest( + candidatesByLine.get(line) ?? [], + (upstream) => comparableVersion(withoutVariant(service, upstream)) !== undefined, + ); + if (best !== undefined) { + updates.push( + buildUpdate("add", service, line, hasLines, undefined, best.upstream, best.revision), + ); + } + } + return { updates, warnings }; } @@ -1005,6 +1079,20 @@ export async function refreshCatalogPin(input: { input.io, ), }; + if (entry.kind === "new-line") { + return { + source: + input.catalog.slice(0, entry.insertAt) + + `, ${JSON.stringify(pin.upstreamVersion)}: ${serializePin(pin)}` + + input.catalog.slice(entry.insertAt), + update: { + service: input.service, + version: resolvedUpstream, + revision: resolution.pin.revision, + target: "additional", + }, + }; + } const written = writePin(input.catalog, entry, pin); if (!written.changed) return { source: input.catalog }; return { @@ -1173,21 +1261,27 @@ async function runManual(argv: ReadonlyArray): Promise { } await Bun.write(catalogPath, result.source); console.log( - `Pinned ${service} ${result.update.target} ${result.update.previousVersion} -> ${result.update.version} r${result.update.revision}.`, + result.update.previousVersion === undefined + ? `Added ${service} ${result.update.target} ${result.update.version} r${result.update.revision}.` + : `Pinned ${service} ${result.update.target} ${result.update.previousVersion} -> ${result.update.version} r${result.update.revision}.`, ); } /** * Line-oriented encoding of one `SlimUpdate`, for a bash loop: `kind`, `branch`, `title` and * `release` (the loop's four required fields, driving the checkout/pin/commit/PR steps) plus - * `from` (the release replaced, for the PR body's "from -> to"). Every field is guaranteed - * non-empty by construction. `\x1f` (unit separator) is the delimiter, not a tab: a title can - * carry ordinary whitespace, and `IFS=$'\t' read` would collapse it. + * `from` (the release replaced, for the PR body's "from -> to"; empty for `add`). Every other + * field is guaranteed non-empty by construction. `\x1f` (unit separator) is the delimiter, not a + * tab: a title can carry ordinary whitespace, and `IFS=$'\t' read` would collapse it. */ function updateLine(update: SlimUpdate): string { - return [update.kind, update.branch, update.title, update.toRelease, update.fromRelease].join( - "\x1f", - ); + return [ + update.kind, + update.branch, + update.title, + update.toRelease, + update.fromRelease ?? "", + ].join("\x1f"); } /** diff --git a/.github/workflows/slim-release-published.yml b/.github/workflows/slim-release-published.yml index 449297676d..41dd0fdf48 100644 --- a/.github/workflows/slim-release-published.yml +++ b/.github/workflows/slim-release-published.yml @@ -136,7 +136,8 @@ jobs: # Reads records on fd 3, not stdin, so commands the loop runs (`bun`, `gh`, `git`) never # consume bytes meant for `read`. while IFS=$'\x1f' read -r -u 3 kind branch title release from; do - if [ -z "$kind" ] || [ -z "$branch" ] || [ -z "$title" ] || [ -z "$release" ] || [ -z "$from" ]; then + # `from` is empty only for `add`, which pins a line's first release. + if [ -z "$kind" ] || [ -z "$branch" ] || [ -z "$title" ] || [ -z "$release" ] || { [ -z "$from" ] && [ "$kind" != "add" ]; }; then echo "::error ::Malformed plan-updates line: kind='${kind}' branch='${branch}' title='${title}' release='${release}' from='${from}'." exit 1 fi @@ -215,10 +216,15 @@ jobs: else action="pins" fi + if [ "$kind" = "add" ]; then + change="adds ${SERVICE} ${release} as a new release line" + else + change="${action} ${SERVICE} from ${from} to ${release}" + fi # Names the release this PR actually pins, not necessarily RELEASE_VERSION: postgres # can plan a hotfix on one line and an unrelated upgrade on another in one dispatch. - body="$(printf 'This %s %s from %s to %s.\n\nhttps://github.com/supabase/slim-services/releases/tag/%s-%s\n\nPlanned after supabase/slim-services published %s. This branch is rewritten from %s whenever a newer relevant release arrives.\n' \ - "$action" "$SERVICE" "$from" "$release" "$SERVICE" "$release" "$RELEASE_VERSION" "$DEFAULT_BRANCH")" + body="$(printf 'This %s.\n\nhttps://github.com/supabase/slim-services/releases/tag/%s-%s\n\nPlanned after supabase/slim-services published %s. This branch is rewritten from %s whenever a newer relevant release arrives.\n' \ + "$change" "$SERVICE" "$release" "$RELEASE_VERSION" "$DEFAULT_BRANCH")" # `--head` matches by branch name only and ignores the owner, so a fork PR with the # same head branch name would otherwise match too; `isCrossRepository` excludes it. diff --git a/apps/cli/docs/stack-commands.md b/apps/cli/docs/stack-commands.md index dd8cbdd896..c53b4a9c07 100644 --- a/apps/cli/docs/stack-commands.md +++ b/apps/cli/docs/stack-commands.md @@ -160,8 +160,9 @@ Other values are rejected. The override is applied before reading the project co `supabase services` follows the same backend selection. In stack mode it lists image versions and canonical `ghcr.io/supabase/cli/...` names from the installed CLI's artifact catalog, including -Mailpit and Vector. PostgreSQL uses the selected major version (15 or 17); invalid configuration -or an unsupported PostgreSQL major warns with the cause and uses catalog defaults. This inventory describes the +Mailpit and Vector. PostgreSQL uses the selected major version (15 or 17), or the catalog's +OrioleDB build when `db.orioledb_version` is set; invalid configuration, an unsupported PostgreSQL +major, or an OrioleDB version the catalog does not pin warns with the cause and uses catalog defaults. This inventory describes the CLI catalog, not running containers, downloaded images, or service health. The Docker and native stack runtimes use the same catalog versions, though a running stack launched by another CLI version or using mirrored images may differ. Legacy version pins and slim-image overrides do not @@ -175,8 +176,9 @@ and without the stack flag. Blank `supabase bootstrap` uses the same scaffold. When the flag is on, `--local` targets of the `db`, `migration`, `test db`, `gen types`, and `inspect` families use the project stack and provision a throwaway shadow database owned by the -stack runtime. Top-level `supabase pull` uses the same stack shadow -as `db pull`. Linked and `--db-url` targets stay on the Management API for engine selection. +stack runtime. The shadow runs the project's resolved database version, the OrioleDB build when +`db.orioledb_version` is set, and its baseline cache is keyed by that version. Top-level +`supabase pull` uses the same stack shadow as `db pull`. Linked and `--db-url` targets stay on the Management API for engine selection. A `--db-url` that matches `config.toml` host and port is still rewritten like a published stack target for dump's tool container. Compose names (`supabase_db_*`, `supabase_network_*`, `db:5432`) are not used. The stack backend requires the in-process pg-delta engine; diff --git a/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts b/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts index f0e554551c..7a3381fd86 100644 --- a/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts +++ b/apps/cli/scripts/render-service-dockerfile.rules.unit.test.ts @@ -6,7 +6,7 @@ import { renderDockerfile } from "./render-service-dockerfile.ts"; // one-line-per-alias check is satisfied by default; each test only mutates what it's // exercising. `vi.mock` factories are hoisted above every other top-level statement, so the // fixture pins are built inline here rather than imported from an outer module. -vi.mock("@supabase/stack/internal/artifacts", () => { +vi.mock("@supabase/stack/internal/artifacts", async (importOriginal) => { const nativePin = { archive: "a".repeat(64), manifest: "b".repeat(64) }; const natives = { "darwin-arm64": nativePin, "linux-amd64": nativePin, "linux-arm64": nativePin }; const pin = (sourceService: string, upstreamImage: string, isDefault: boolean) => ({ @@ -22,9 +22,11 @@ vi.mock("@supabase/stack/internal/artifacts", () => { }, }); return { + ...(await importOriginal()), catalogPins: () => [ pin("postgres", "supabase/postgres:17.0.0", true), pin("postgres", "supabase/postgres:15.0.0", false), + pin("postgres", "supabase/postgres:17.0.0-orioledb", false), pin("mailpit", "axllent/mailpit:v1.0.0", true), pin("postgrest", "postgrest/postgrest:v1.0.0", true), pin("pgmeta", "supabase/postgres-meta:v1.0.0", true), @@ -65,10 +67,11 @@ function baseDockerfile(overrides: Readonly> = {}): strin } describe("renderDockerfile: pin selection", () => { - test("pg takes the default pin and pg15 takes the additional one, by isDefault, not a version-string check", () => { + test("pg takes the default pin and pg15 the additional stock pin; an OrioleDB pin has no alias", () => { const rendered = renderDockerfile(baseDockerfile()); expect(rendered).toContain("FROM supabase/postgres:17.0.0 AS pg\n"); expect(rendered).toContain("FROM supabase/postgres:15.0.0 AS pg15\n"); + expect(rendered).not.toContain("orioledb"); }); }); diff --git a/apps/cli/scripts/render-service-dockerfile.ts b/apps/cli/scripts/render-service-dockerfile.ts index 6c6c114eb6..79327facf3 100644 --- a/apps/cli/scripts/render-service-dockerfile.ts +++ b/apps/cli/scripts/render-service-dockerfile.ts @@ -11,7 +11,11 @@ // The `--check` mode is what CI runs (as a `render-service-dockerfile.unit.test.ts` assertion) to // catch a hand-edited generated line, or a catalog change that hasn't been regenerated yet. import { fileURLToPath } from "node:url"; -import { catalogPins, type ArtifactPin } from "@supabase/stack/internal/artifacts"; +import { + catalogPins, + isOrioledbVersion, + type ArtifactPin, +} from "@supabase/stack/internal/artifacts"; // Resolved from this module's own URL, so both the CLI invocation (cwd = repo root) and the // vitest unit test (cwd = apps/cli) read the same files. @@ -37,7 +41,10 @@ const REPOSITORY_OVERRIDES: Readonly> = { interface SlimAliasSpec { readonly sourceService: string; - /** Selects the catalog's default pin (every alias but `pg15`) or its lone additional pin. */ + /** + * Selects the catalog's default pin (every alias but `pg15`) or its lone additional stock pin; + * OrioleDB pins have no Dockerfile alias. + */ readonly wantDefault: boolean; } @@ -65,7 +72,10 @@ function selectPin( pins: ReadonlyArray, ): ArtifactPin { const candidates = pins.filter( - (entry) => entry.sourceService === spec.sourceService && entry.isDefault === spec.wantDefault, + (entry) => + entry.sourceService === spec.sourceService && + entry.isDefault === spec.wantDefault && + !isOrioledbVersion(entry.pin.upstreamVersion), ); if (candidates.length !== 1) { throw new Error( diff --git a/apps/cli/src/command-internal/db-image.ts b/apps/cli/src/command-internal/db-image.ts index 8db35f7ded..7436a030a8 100644 --- a/apps/cli/src/command-internal/db-image.ts +++ b/apps/cli/src/command-internal/db-image.ts @@ -1,7 +1,11 @@ import { Effect, type FileSystem, type Path } from "effect"; import { dockerfileServiceImageRaw } from "../shared/services/dockerfile-images.ts"; import { postgresImageForDbMajorVersion } from "../shared/services/services.shared.ts"; -import { slimImageForCurrentPin, slimImagesEnabled } from "../shared/services/slim-images.ts"; +import { + slimImageForAlias, + slimImageForCurrentPin, + slimImagesEnabled, +} from "../shared/services/slim-images.ts"; import { PropOrioleDb } from "../shared/telemetry/event-catalog.ts"; import { recordCommandTelemetry } from "../telemetry/command-telemetry-attributes.ts"; @@ -77,6 +81,7 @@ export const resolveDbImage = Effect.fnUntraced(function* ( majorVersion: number, orioledbVersion?: string, ) { + const slim = yield* slimImagesEnabled; yield* recordOrioleDbTelemetry(orioledbVersion, majorVersion); // The OrioleDB tag takes precedence over the default/pinned image. if (selectsOrioleDb(orioledbVersion, majorVersion)) { @@ -84,9 +89,8 @@ export const resolveDbImage = Effect.fnUntraced(function* ( versionCompare(orioledbVersion, "15.1.1.13") > 0 ? `supabase/postgres:${orioledbVersion}-orioledb` : `supabase/postgres:orioledb-${orioledbVersion}`; - return { image, configImage: image }; + return { image: slimImageForAlias("pg", image, slim), configImage: image }; } - const slim = yield* slimImagesEnabled; const currentRaw = postgresImageForDbMajorVersion(majorVersion) ?? pgImageRaw(); let appliedPin: string | undefined; if (majorVersion > 14) { diff --git a/apps/cli/src/command-internal/db-image.unit.test.ts b/apps/cli/src/command-internal/db-image.unit.test.ts index d66aabd1b2..d79473e4ec 100644 --- a/apps/cli/src/command-internal/db-image.unit.test.ts +++ b/apps/cli/src/command-internal/db-image.unit.test.ts @@ -113,6 +113,18 @@ describe("resolveDbImage", () => { }); }); + it.effect("keeps an OrioleDB tag the catalog does not pin on docker.io", () => { + vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); + const dir = withTemp(); + return Effect.gen(function* () { + expect(yield* resolve(dir, 17, "17.0.0.000")).toEqual({ + image: "supabase/postgres:17.0.0.000-orioledb", + configImage: "supabase/postgres:17.0.0.000-orioledb", + }); + rmSync(dir, { recursive: true, force: true }); + }); + }); + it.effect("rewrites the current PG15 default tag to its catalog-pinned slim image", () => { vi.stubEnv("SUPABASE_USE_SLIM_IMAGES", "true"); const dir = withTemp(); diff --git a/apps/cli/src/command-internal/stack-config.ts b/apps/cli/src/command-internal/stack-config.ts index bf8ddcd138..db9642bfc4 100644 --- a/apps/cli/src/command-internal/stack-config.ts +++ b/apps/cli/src/command-internal/stack-config.ts @@ -8,7 +8,17 @@ import { DEFAULT_SIGNING_KEY, } from "@supabase/stack/defaults"; import { type ServiceCreationInput as ServiceCreationType } from "@supabase/stack/effect"; -import { Crypto, Effect, Data, FileSystem, Path, Redacted, Schema, SchemaIssue } from "effect"; +import { + Crypto, + Effect, + Data, + FileSystem, + Path, + Redacted, + Result, + Schema, + SchemaIssue, +} from "effect"; import { FetchHttpClient } from "effect/unstable/http"; import { loadLocalProjectContext, type LocalProjectContext } from "./local-project-context.ts"; @@ -17,6 +27,7 @@ import { CLI_VERSION } from "../shared/cli/version.ts"; import { resolveAuthConfig } from "./stack-auth-config.ts"; import { parseGoDuration } from "./go-duration.ts"; import { parseFileSizeLimit } from "./storage-bucket-config.ts"; +import { stackDatabaseVersion } from "./stack-database-version.ts"; import { decryptAuthSecret, @@ -298,6 +309,12 @@ export const stackMajorVersionSetting: StackEndpointSetting = { envVar: "SUPABASE_DB_MAJOR_VERSION", }; +/** `db.orioledb_version`'s config key and `SUPABASE_DB_ORIOLEDB_VERSION` override. */ +export const stackOrioledbVersionSetting: StackEndpointSetting = { + configPath: "db.orioledb_version", + envVar: "SUPABASE_DB_ORIOLEDB_VERSION", +}; + const authProviderNames = [ "apple", "azure", @@ -801,7 +818,7 @@ const resolveEffectiveCliConfig = ( port: resolvedPort("SUPABASE_DB_PORT", db.port, "db.port", env), major_version: envOverrideMajorVersion(db.major_version, env), health_timeout: envOverride("SUPABASE_DB_HEALTH_TIMEOUT", db.health_timeout, env), - orioledb_version: envOverride("SUPABASE_DB_ORIOLEDB_VERSION", db.orioledb_version, env), + orioledb_version: envOverride(stackOrioledbVersionSetting.envVar, db.orioledb_version, env), settings: resolveDbSettingsEnvOverrides(db.settings, env), pooler: resolvedPooler, }, @@ -867,7 +884,6 @@ const unsupportedConfigPaths = [ { path: "analytics.gcp_jwt_path", active: (config: CliConfig) => config.analytics.enabled }, { path: "edge_runtime.deno_version", active: (config: CliConfig) => config.edge_runtime.enabled }, { path: "storage.analytics", active: (config: CliConfig) => config.storage.enabled }, - { path: "db.orioledb_version", active: (_config: CliConfig) => true }, ] as const; const pathValue = (value: unknown, path: string): unknown => { @@ -996,6 +1012,10 @@ export const loadStackConfig = Effect.fn("StackConfig.load")( const validationError = configValidationError(validatedConfig); if (validationError !== undefined) return yield* new StackConfigError({ message: validationError }); + const databaseVersionResult = stackDatabaseVersion(validatedConfig.db); + if (Result.isFailure(databaseVersionResult)) + return yield* new StackConfigError({ message: databaseVersionResult.failure }); + const databaseVersion = databaseVersionResult.success; const externalProviders = yield* Effect.try({ try: () => @@ -1270,7 +1290,7 @@ export const loadStackConfig = Effect.fn("StackConfig.load")( { service: "database", config: { - version: String(validatedConfig.db.major_version), + version: databaseVersion, ...(jwtSecret === undefined ? {} : { jwtSecret }), jwtExpiry: validatedConfig.auth.jwt_expiry, settings: validatedConfig.db.settings, diff --git a/apps/cli/src/command-internal/stack-database-version.ts b/apps/cli/src/command-internal/stack-database-version.ts new file mode 100644 index 0000000000..4760fa2d2b --- /dev/null +++ b/apps/cli/src/command-internal/stack-database-version.ts @@ -0,0 +1,31 @@ +import { + orioledbPostgresVersion, + orioledbVersions, + postgresMajor, +} from "@supabase/stack/internal/artifacts"; +import { Result } from "effect"; + +/** + * Database artifact version the stack runs for a project's `[db]` config: the major alias, or the + * OrioleDB build named by `db.orioledb_version`, which must be pinned in the artifact catalog. + */ +export const stackDatabaseVersion = ( + db: { readonly major_version: number; readonly orioledb_version?: string | undefined }, + supported: ReadonlyArray = orioledbVersions(), +): Result.Result => { + const orioledb = db.orioledb_version; + if (orioledb === undefined || orioledb.length === 0) + return Result.succeed(String(db.major_version)); + if (!supported.includes(orioledb)) + return Result.fail( + `db.orioledb_version = ${orioledb} requires a published OrioleDB artifact; supported OrioleDB versions: ${ + supported.length === 0 ? "none" : supported.join(", ") + }. A saved stack keeps its database version; switching it means recreating the stack with supabase stack destroy, which permanently deletes its local database data`, + ); + const major = postgresMajor(orioledb); + if (major !== String(db.major_version)) + return Result.fail( + `db.orioledb_version = ${orioledb} requires db.major_version = ${major} for the experimental stack`, + ); + return Result.succeed(orioledbPostgresVersion(orioledb)); +}; diff --git a/apps/cli/src/command-internal/stack-database-version.unit.test.ts b/apps/cli/src/command-internal/stack-database-version.unit.test.ts new file mode 100644 index 0000000000..94519befa5 --- /dev/null +++ b/apps/cli/src/command-internal/stack-database-version.unit.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "@effect/vitest"; +import { orioledbVersions, postgresMajor } from "@supabase/stack/internal/artifacts"; +import { Result } from "effect"; + +import { stackDatabaseVersion } from "./stack-database-version.ts"; + +const published = ["17.11.0.002"]; + +describe("stackDatabaseVersion", () => { + it("runs the configured major alias when OrioleDB is unset or empty", () => { + expect(stackDatabaseVersion({ major_version: 17 }, published)).toEqual(Result.succeed("17")); + expect(stackDatabaseVersion({ major_version: 15, orioledb_version: "" }, published)).toEqual( + Result.succeed("15"), + ); + }); + + it("routes a published db.orioledb_version to its OrioleDB artifact", () => { + expect( + stackDatabaseVersion({ major_version: 17, orioledb_version: "17.11.0.002" }, published), + ).toEqual(Result.succeed("17.11.0.002-orioledb")); + }); + + it("routes every OrioleDB version the artifact catalog pins", () => { + const pinned = orioledbVersions(); + expect(pinned.length).toBeGreaterThan(0); + for (const version of pinned) + expect( + stackDatabaseVersion({ + major_version: Number(postgresMajor(version)), + orioledb_version: version, + }), + ).toEqual(Result.succeed(`${version}-orioledb`)); + }); + + it("rejects an OrioleDB version whose major differs from db.major_version", () => { + expect( + stackDatabaseVersion({ major_version: 15, orioledb_version: "17.11.0.002" }, published), + ).toEqual( + Result.fail( + "db.orioledb_version = 17.11.0.002 requires db.major_version = 17 for the experimental stack", + ), + ); + }); + + it("fails closed on an unpublished OrioleDB version, listing the published ones", () => { + expect( + stackDatabaseVersion({ major_version: 17, orioledb_version: "17.6.1.000" }, published), + ).toEqual( + Result.fail( + "db.orioledb_version = 17.6.1.000 requires a published OrioleDB artifact; supported OrioleDB versions: 17.11.0.002. A saved stack keeps its database version; switching it means recreating the stack with supabase stack destroy, which permanently deletes its local database data", + ), + ); + expect( + stackDatabaseVersion({ major_version: 17, orioledb_version: "17.11.0.002" }, []), + ).toEqual( + Result.fail( + "db.orioledb_version = 17.11.0.002 requires a published OrioleDB artifact; supported OrioleDB versions: none. A saved stack keeps its database version; switching it means recreating the stack with supabase stack destroy, which permanently deletes its local database data", + ), + ); + }); +}); diff --git a/apps/cli/src/command-internal/stack-shadow-cache.integration.test.ts b/apps/cli/src/command-internal/stack-shadow-cache.integration.test.ts index d750bdea19..003c982f07 100644 --- a/apps/cli/src/command-internal/stack-shadow-cache.integration.test.ts +++ b/apps/cli/src/command-internal/stack-shadow-cache.integration.test.ts @@ -66,19 +66,27 @@ describe("stack shadow cache entry", () => { yield* fs.makeDirectory(path.join(root, "supabase"), { recursive: true }); const base = input(fs, path, root); - const first = yield* stackShadowCacheEntry(base, "native", "darwin", "arm64", "config"); + const first = yield* stackShadowCacheEntry(base, "17", "native", "darwin", "arm64", "config"); if (first === undefined) return yield* Effect.die("cache entry unexpectedly disabled"); yield* fs.writeFileString( path.join(root, "supabase", "roles.sql"), "CREATE ROLE cache_probe;\n", ); - const withRoles = yield* stackShadowCacheEntry(base, "native", "darwin", "arm64", "config"); + const withRoles = yield* stackShadowCacheEntry( + base, + "17", + "native", + "darwin", + "arm64", + "config", + ); if (withRoles === undefined) return yield* Effect.die("roles entry unexpectedly disabled"); expect(withRoles.key).not.toBe(first.key); const withWebhooks = yield* stackShadowCacheEntry( input(fs, path, root, { webhooksEnabled: true }), + "17", "native", "darwin", "arm64", @@ -90,6 +98,7 @@ describe("stack shadow cache entry", () => { const withPassword = yield* stackShadowCacheEntry( input(fs, path, root, {}, { password: "rotated-password" }), + "17", "native", "darwin", "arm64", @@ -100,11 +109,12 @@ describe("stack shadow cache entry", () => { expect(withPassword.key).not.toBe(withRoles.key); expect( - yield* stackShadowCacheEntry(base, "native", "darwin", "arm64", "config", true), + yield* stackShadowCacheEntry(base, "17", "native", "darwin", "arm64", "config", true), ).toBeUndefined(); expect( yield* stackShadowCacheEntry( input(fs, path, root, { projectEnvValues: { SUPABASE_SHADOW_CACHE: "0" } }), + "17", "native", "darwin", "arm64", @@ -113,4 +123,24 @@ describe("stack shadow cache entry", () => { ).toBeUndefined(); }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); + + it.effect("keys on the database version the shadow runs, not the configured major", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-cache-version-" }); + const base = input(fs, path, root); + + const stock = yield* stackShadowCacheEntry(base, "17", "native", "darwin", "arm64", "config"); + const other = yield* stackShadowCacheEntry(base, "15", "native", "darwin", "arm64", "config"); + expect(stock?.key).toBeDefined(); + expect(other?.key).toBeDefined(); + expect(other?.key).not.toBe(stock?.key); + + const unpinned = yield* Effect.flip( + stackShadowCacheEntry(base, "17.0.0.000-orioledb", "native", "darwin", "arm64", "config"), + ); + expect(unpinned.message).toContain("17.0.0.000-orioledb"); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); }); diff --git a/apps/cli/src/command-internal/stack-shadow-cache.ts b/apps/cli/src/command-internal/stack-shadow-cache.ts index 0249891d0d..827acb043c 100644 --- a/apps/cli/src/command-internal/stack-shadow-cache.ts +++ b/apps/cli/src/command-internal/stack-shadow-cache.ts @@ -23,6 +23,7 @@ const readRoles = (input: ShadowSetupInput) => export const stackShadowCacheEntry = Effect.fn("StackShadowCache.entry")(function* ( input: ShadowSetupInput, + databaseVersion: string, runtime: string, platform: string, arch: string, @@ -39,7 +40,7 @@ export const stackShadowCacheEntry = Effect.fn("StackShadowCache.entry")(functio const rolesSql = yield* readRoles(input); const postgres = yield* resolveArtifact({ service: "database", - version: postgresVersion(String(input.setup.majorVersion)), + version: postgresVersion(databaseVersion), }); const image = (service: "auth" | "storage" | "realtime", enabled: boolean) => enabled ? resolveArtifact({ service }) : Effect.succeed({ image: "", version: "" }); diff --git a/apps/cli/src/command-internal/stack-shadow.integration.test.ts b/apps/cli/src/command-internal/stack-shadow.integration.test.ts index 40b038279b..faacbc5e35 100644 --- a/apps/cli/src/command-internal/stack-shadow.integration.test.ts +++ b/apps/cli/src/command-internal/stack-shadow.integration.test.ts @@ -515,4 +515,39 @@ describe("stack shadow databases", () => { }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), 120_000, ); + + it.live("refuses an unpinned OrioleDB version before creating a shadow stack", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-shadow-orioledb-" }); + const output = mockOutput(); + const roots: string[] = []; + const setup = { + ...input(fs, path, root), + db: { major_version: 17, orioledb_version: "17.0.0.000", settings: {} }, + }; + + const error = yield* stackWithShadowDatabase(setup, () => Effect.void, { + runtime: "native", + }).pipe( + Effect.provide( + Layer.mergeAll( + recordingApi(roots, []), + stackCatalogSetupLayer, + dbConnectionLayer, + runtimeInfoLayer, + mockCommandSettings({ workdir: root, supabaseHome: root }), + output.layer, + ), + ), + Effect.flip, + ); + + expect(error.message).toContain( + "db.orioledb_version = 17.0.0.000 requires a published OrioleDB artifact", + ); + expect(roots).toEqual([]); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); }); diff --git a/apps/cli/src/command-internal/stack-shadow.ts b/apps/cli/src/command-internal/stack-shadow.ts index 86c2ee6276..6284e74f2b 100644 --- a/apps/cli/src/command-internal/stack-shadow.ts +++ b/apps/cli/src/command-internal/stack-shadow.ts @@ -18,6 +18,7 @@ import type { SetupDatabaseOptions } from "./db-bootstrap/db-setup.ts"; import { listLocalMigrationPaths } from "./migration-history.ts"; import { applyMigrations } from "./migration-apply.ts"; import { stackShadowCacheEntry, stackShadowCacheRoles } from "./stack-shadow-cache.ts"; +import { stackDatabaseVersion } from "./stack-database-version.ts"; type Runtime = "native" | "docker" | "podman"; @@ -79,9 +80,21 @@ const acquireNamespace = Effect.fn("StackShadow.acquireNamespace")(function* (op return { stack, runtime }; }); +/** The project's resolved database version, so the shadow runs the same line as the local database. */ +const shadowDatabaseVersion = (input: ShadowSetupInput) => + Effect.fromResult( + stackDatabaseVersion({ + major_version: input.setup.majorVersion, + orioledb_version: input.db.orioledb_version, + }), + ).pipe( + Effect.mapError((message) => new ShadowDbError({ message, reason: "container_configuration" })), + ); + const initialize = Effect.fn("StackShadow.initialize")(function* ( stack: Stack, runtime: Runtime, + version: string, input: ShadowSetupInput, opts: ShadowOptions, ) { @@ -97,7 +110,7 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( stack.services.create({ service: "database", config: { - version: String(input.setup.majorVersion), + version, databasePassword: Redacted.make(input.password), jwtSecret: Redacted.make(input.jwtSecret), jwtExpiry: input.jwtExpiry, @@ -113,6 +126,7 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( const cacheResolution = yield* Effect.result( stackShadowCacheEntry( input, + version, runtime, runtimeInfo.platform, runtimeInfo.arch, @@ -218,7 +232,7 @@ const initialize = Effect.fn("StackShadow.initialize")(function* ( host: conn.host, port: conn.port, runtime, - version: String(input.setup.majorVersion), + version, snapshotKey, restoredFromSnapshot: restored, } satisfies StackShadowAcquiredHandle; @@ -233,6 +247,7 @@ export const stackAcquireShadowDatabase = Effect.fn("StackShadow.acquire")(funct opts: ShadowOptions = {}, ) { const output = yield* Output; + const version = yield* shadowDatabaseVersion(input); const namespace = yield* Effect.acquireRelease(acquireNamespace(opts), ({ stack }) => stack.destroy.pipe( Effect.flatMap((result) => @@ -248,7 +263,7 @@ export const stackAcquireShadowDatabase = Effect.fn("StackShadow.acquire")(funct ), ), ); - return yield* initialize(namespace.stack, namespace.runtime, input, opts); + return yield* initialize(namespace.stack, namespace.runtime, version, input, opts); }, Effect.mapError(shadowError)); /** Runs a command against a fresh shadow, then destroys its owned namespace. */ diff --git a/apps/cli/src/command-internal/test-db.handler.ts b/apps/cli/src/command-internal/test-db.handler.ts index f5010ec342..efa906d18d 100644 --- a/apps/cli/src/command-internal/test-db.handler.ts +++ b/apps/cli/src/command-internal/test-db.handler.ts @@ -22,7 +22,7 @@ import { TestDbNoTestsError, TestDbRunError, } from "./test-db.errors.ts"; -import { buildPgProveArgs } from "./test-db.pg-prove-args.ts"; +import { buildPgProveArgs, pgProveMajor } from "./test-db.pg-prove-args.ts"; import { currentStackBackend } from "./stack-backend.ts"; import { rewriteDumpHostForToolContainer, @@ -79,15 +79,10 @@ const managedStackFor = Effect.fn("test.db.managedStack")(function* () { return yield* new LocalDbRunningError({ message: "The local stack primary service is not a database.", }); - const major = - status.config.config.version === "15" - ? 15 - : status.config.config.version === "17" - ? 17 - : undefined; + const major = pgProveMajor(status.config.config.version); if (major === undefined) return yield* new LocalDbRunningError({ - message: `The local database major version ${status.config.config.version} is not supported by pg_prove.`, + message: `The local database version ${status.config.config.version} is not supported by pg_prove.`, }); const credentials = yield* database.credentials({ from: "runtime" }).pipe( Effect.mapError( diff --git a/apps/cli/src/command-internal/test-db.pg-prove-args.ts b/apps/cli/src/command-internal/test-db.pg-prove-args.ts index 95e8c232dc..1e7a820e1a 100644 --- a/apps/cli/src/command-internal/test-db.pg-prove-args.ts +++ b/apps/cli/src/command-internal/test-db.pg-prove-args.ts @@ -1,5 +1,6 @@ import * as nodePath from "node:path"; import { Option } from "effect"; +import { postgresMajor } from "@supabase/stack/internal/artifacts"; import { toDockerMountPath } from "./docker-path.ts"; interface PostgresClientMount { @@ -24,6 +25,12 @@ export interface PgProveArgs { readonly workingDir: Option.Option; } +/** Catalog `pg_prove` major for a stack database artifact version, including OrioleDB builds. */ +export const pgProveMajor = (version: string): 15 | 17 | undefined => { + const major = postgresMajor(version); + return major === "15" ? 15 : major === "17" ? 17 : undefined; +}; + /** * Builds the `pg_prove` command, volume binds, and working directory for a * `test db` run. diff --git a/apps/cli/src/command-internal/test-db.pg-prove-args.unit.test.ts b/apps/cli/src/command-internal/test-db.pg-prove-args.unit.test.ts index a2da1d0aa1..913d645913 100644 --- a/apps/cli/src/command-internal/test-db.pg-prove-args.unit.test.ts +++ b/apps/cli/src/command-internal/test-db.pg-prove-args.unit.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "vitest"; import { Option } from "effect"; -import { buildPgProveArgs } from "./test-db.pg-prove-args.ts"; +import { buildPgProveArgs, pgProveMajor } from "./test-db.pg-prove-args.ts"; describe("buildPgProveArgs", () => { test("defaults to /supabase/tests when no paths are given", () => { @@ -95,3 +95,16 @@ describe("buildPgProveArgs", () => { expect(result.cmd.at(-1)).toBe("--verbose"); }); }); + +describe("pgProveMajor", () => { + test("maps stock major aliases and OrioleDB builds to their PostgreSQL major", () => { + expect(pgProveMajor("15")).toBe(15); + expect(pgProveMajor("17")).toBe(17); + expect(pgProveMajor("17.11.0.002-orioledb")).toBe(17); + }); + + test("rejects majors without a catalog pg_prove", () => { + expect(pgProveMajor("14")).toBeUndefined(); + expect(pgProveMajor("16.4.1.000-orioledb")).toBeUndefined(); + }); +}); diff --git a/apps/cli/src/commands/db/dump/SIDE_EFFECTS.md b/apps/cli/src/commands/db/dump/SIDE_EFFECTS.md index ba27b252c2..9e4f37646e 100644 --- a/apps/cli/src/commands/db/dump/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/dump/SIDE_EFFECTS.md @@ -33,15 +33,15 @@ image), to stdout or `--file`. ## Environment Variables -| Variable | Purpose | -| ----------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `SUPABASE_DB_PASSWORD` (`DB_PASSWORD` viper key; `--password`/`-p` overrides) | remote DB password | -| `SUPABASE_ACCESS_TOKEN` | `--linked` auth | -| `BITBUCKET_CLONE_DIR` | (no-op for dump — no `--security-opt` is set) | -| `SUPABASE_INTERNAL_IMAGE_REGISTRY` | rewrite the pg image registry for compose dumps; stack dumps use the catalog image pin unchanged | -| `SUPABASE_USE_SLIM_IMAGES` | resolve the current Postgres pin from the slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); majors 13/15 use `15.14.1.167` when the flag is on; historical pins, PG14, OrioleDB, and flag-off `15.8.1.085` stay on docker.io | -| `DOCKER_HOST` | docker daemon endpoint | -| `MSYSTEM`, `TERM_PROGRAM` | suppress the piped-stdout non-ASCII warning in MSYS/mintty sessions | +| Variable | Purpose | +| ----------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `SUPABASE_DB_PASSWORD` (`DB_PASSWORD` viper key; `--password`/`-p` overrides) | remote DB password | +| `SUPABASE_ACCESS_TOKEN` | `--linked` auth | +| `BITBUCKET_CLONE_DIR` | (no-op for dump — no `--security-opt` is set) | +| `SUPABASE_INTERNAL_IMAGE_REGISTRY` | rewrite the pg image registry for compose dumps; stack dumps use the catalog image pin unchanged | +| `SUPABASE_USE_SLIM_IMAGES` | resolve the current Postgres pin from the slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); majors 13/15 use `15.14.1.167` when the flag is on; historical pins, PG14, OrioleDB tags the catalog does not pin, and flag-off `15.8.1.085` stay on docker.io | +| `DOCKER_HOST` | docker daemon endpoint | +| `MSYSTEM`, `TERM_PROGRAM` | suppress the piped-stdout non-ASCII warning in MSYS/mintty sessions | ## Exit Codes diff --git a/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md b/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md index 07ae92efc8..ac893527b4 100644 --- a/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/pull/SIDE_EFFECTS.md @@ -122,19 +122,19 @@ at all, so nothing is cached for it. ## Environment Variables -| Variable | Purpose | Required? | -| ------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | -| `SUPABASE_ACCESS_TOKEN` | auth for the linked target | no | -| `SUPABASE_DB_PASSWORD` | remote DB password (overridden by `-p`) | no | -| `SUPABASE_DB_SHADOW_PORT` | shadow container's host port (`db.shadow_port`) — NOT `SUPABASE_DB_PORT`, which the shadow never reads | no | -| `SUPABASE_DB_MAJOR_VERSION` / `SUPABASE_DB_HEALTH_TIMEOUT` / `SUPABASE_DB_SETTINGS_*` | shadow container-config overrides, same as `db start`/`db reset` | no | -| `SUPABASE_PROJECT_ID` | overrides the shadow container's project id/labels, same as `db start`/`db reset` (`utils.DbId`); ALSO the linked-ref resolution fallback `--project-ref` supersedes — see Notes for the narrower scope of the flag | no | -| `SUPABASE_NETWORK_ID` (`--network-id`) | forces the shadow and initial-migra `pg_dump` containers onto an existing Docker network | no | -| `SUPABASE_USE_SLIM_IMAGES` | resolves the current-pin shadow Postgres, `pg_dump`, PG15+ realtime/storage/auth migrate-job images (migration-style cold shadow), and (for migra) the edge-runtime image from the slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); majors 13/15 use `15.14.1.167` when the flag is on; historical pins, PG14, OrioleDB, flag-off `15.8.1.085`, and `deno_version = 1` stay on docker.io | no | -| `SUPABASE_HOME` | overrides the `~/.supabase` root used for the shadow baseline cache (and other CLI state) | no | -| `SUPABASE_SHADOW_CACHE` | shadow baseline cache; on by default, opt-out (`0`/`false`); the shadow's post-baseline state is saved under a managed snapshot key and restored into the next run's fresh stack database (see Notes) | no | -| `SUPABASE_EXPERIMENTAL_PG_DELTA` | force pg-delta diff engine | no | -| `SUPABASE_EXPERIMENTAL` | selects the deprecated in-process structured-dump export (same as `--declarative`) when `--declarative` is not set | no | +| Variable | Purpose | Required? | +| ------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | +| `SUPABASE_ACCESS_TOKEN` | auth for the linked target | no | +| `SUPABASE_DB_PASSWORD` | remote DB password (overridden by `-p`) | no | +| `SUPABASE_DB_SHADOW_PORT` | shadow container's host port (`db.shadow_port`) — NOT `SUPABASE_DB_PORT`, which the shadow never reads | no | +| `SUPABASE_DB_MAJOR_VERSION` / `SUPABASE_DB_HEALTH_TIMEOUT` / `SUPABASE_DB_SETTINGS_*` | shadow container-config overrides, same as `db start`/`db reset` | no | +| `SUPABASE_PROJECT_ID` | overrides the shadow container's project id/labels, same as `db start`/`db reset` (`utils.DbId`); ALSO the linked-ref resolution fallback `--project-ref` supersedes — see Notes for the narrower scope of the flag | no | +| `SUPABASE_NETWORK_ID` (`--network-id`) | forces the shadow and initial-migra `pg_dump` containers onto an existing Docker network | no | +| `SUPABASE_USE_SLIM_IMAGES` | resolves the current-pin shadow Postgres, `pg_dump`, PG15+ realtime/storage/auth migrate-job images (migration-style cold shadow), and (for migra) the edge-runtime image from the slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); majors 13/15 use `15.14.1.167` when the flag is on; historical pins, PG14, OrioleDB tags the catalog does not pin, flag-off `15.8.1.085`, and `deno_version = 1` stay on docker.io | no | +| `SUPABASE_HOME` | overrides the `~/.supabase` root used for the shadow baseline cache (and other CLI state) | no | +| `SUPABASE_SHADOW_CACHE` | shadow baseline cache; on by default, opt-out (`0`/`false`); the shadow's post-baseline state is saved under a managed snapshot key and restored into the next run's fresh stack database (see Notes) | no | +| `SUPABASE_EXPERIMENTAL_PG_DELTA` | force pg-delta diff engine | no | +| `SUPABASE_EXPERIMENTAL` | selects the deprecated in-process structured-dump export (same as `--declarative`) when `--declarative` is not set | no | ## Exit Codes diff --git a/apps/cli/src/commands/db/reset/SIDE_EFFECTS.md b/apps/cli/src/commands/db/reset/SIDE_EFFECTS.md index 0eb54c6e9b..6e27d96771 100644 --- a/apps/cli/src/commands/db/reset/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/reset/SIDE_EFFECTS.md @@ -171,7 +171,7 @@ the whole reset** (not just "skip buckets"). | `SUPABASE_DB_MIGRATIONS_SCHEMA_PATHS` | overrides `[db.migrations].schema_paths` (viper `AutomaticEnv`, beats the config-file value) for the schema-files apply branch — genuinely effective on both targets now | no (no dedicated flag — config-file-only otherwise) | | `SUPABASE_PROJECT_ID` | overrides the local container id; ALSO the linked-ref resolution fallback `--project-ref` supersedes — see Notes for the narrower scope of the flag | no | | `SUPABASE_INTERNAL_IMAGE_REGISTRY` | overrides the image registry used to resolve the local path's container images (project `.env` or shell) | no (project `.env` or shell) | -| `SUPABASE_USE_SLIM_IMAGES` | resolves the local-reset Postgres image and the realtime/storage/auth migrate-job images from slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); majors 13/15 use `15.14.1.167` when the flag is on; historical pins, PG14, OrioleDB, and flag-off `15.8.1.085` stay on docker.io | no (ambient shell only) | +| `SUPABASE_USE_SLIM_IMAGES` | resolves the local-reset Postgres image and the realtime/storage/auth migrate-job images from slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); majors 13/15 use `15.14.1.167` when the flag is on; historical pins, PG14, OrioleDB tags the catalog does not pin, and flag-off `15.8.1.085` stay on docker.io | no (ambient shell only) | | `SUPABASE_DB_PORT` / `SUPABASE_DB_MAJOR_VERSION` / `SUPABASE_DB_HEALTH_TIMEOUT` / `SUPABASE_DB_SETTINGS_*` | local-path container-recreate config overrides, same as `db start` | no | | `SUPABASE_API_PORT` / `SUPABASE_API_EXTERNAL_URL` / `SUPABASE_API_TLS_*` / `SUPABASE_API_ENABLED` | legacy-backend bucket-seed step only: override the matching `[api]` fields for the Storage gateway URL/TLS, same as `seed buckets` (shell or project dotenv; #6452). Not consulted on the stack backend (see above) | no | | `SUPABASE_AUTH_JWT_SECRET` / `SUPABASE_AUTH_SERVICE_ROLE_KEY` | local path: override `auth.jwt_secret` / `auth.service_role_key` (shell or project dotenv, `encrypted:` decrypted) — feeds the recreated Postgres container (jwt secret), the fresh-database setup jobs (both) and the legacy-backend bucket-seed Storage service-role key, same as `seed buckets`. Not consulted on the stack backend, which uses the stack's own service-role JWT | no | diff --git a/apps/cli/src/commands/db/schema/declarative/generate/SIDE_EFFECTS.md b/apps/cli/src/commands/db/schema/declarative/generate/SIDE_EFFECTS.md index 86ad6ef40a..30ed404763 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/schema/declarative/generate/SIDE_EFFECTS.md @@ -41,15 +41,15 @@ formatting without disabling safe compaction. ## Environment Variables -| Variable | Purpose | Required? | -| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | -| `SUPABASE_ACCESS_TOKEN` | auth token for `--linked` | no | -| `DB_PASSWORD` | password for `--linked` / `--db-url` | no | -| `SUPABASE_HOME` | overrides the `~/.supabase` root (access token and other CLI state) | no | -| `PGDELTA_DEBUG` | bundled-engine debug artifacts | no | -| `SUPABASE_SERVICES_HOSTNAME` | local DB host for `--local` | no | -| `DOCKER_HOST` | tcp daemon host used as the local DB host fallback | no | -| `SUPABASE_USE_SLIM_IMAGES` | resolves the expected local `db` image for the stale-container guard from the slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); majors 13/15 use `15.14.1.167` when the flag is on; historical pins, PG14, OrioleDB, and flag-off `15.8.1.085` stay on docker.io | no | +| Variable | Purpose | Required? | +| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | +| `SUPABASE_ACCESS_TOKEN` | auth token for `--linked` | no | +| `DB_PASSWORD` | password for `--linked` / `--db-url` | no | +| `SUPABASE_HOME` | overrides the `~/.supabase` root (access token and other CLI state) | no | +| `PGDELTA_DEBUG` | bundled-engine debug artifacts | no | +| `SUPABASE_SERVICES_HOSTNAME` | local DB host for `--local` | no | +| `DOCKER_HOST` | tcp daemon host used as the local DB host fallback | no | +| `SUPABASE_USE_SLIM_IMAGES` | resolves the expected local `db` image for the stale-container guard from the slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); majors 13/15 use `15.14.1.167` when the flag is on; historical pins, PG14, OrioleDB tags the catalog does not pin, and flag-off `15.8.1.085` stay on docker.io | no | ## Exit Codes diff --git a/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md b/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md index d9fb9bdfd1..55188fe177 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/schema/declarative/sync/SIDE_EFFECTS.md @@ -53,14 +53,14 @@ disabling safe compaction. ## Environment Variables -| Variable | Purpose | Required? | -| ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | -| `SUPABASE_HOME` | overrides the `~/.supabase` root used for the shadow baseline cache (and other CLI state) | no | -| `SUPABASE_SHADOW_CACHE` | shadow baseline cache; on by default, opt-out (`0`/`false`); the shadow's post-baseline state is saved under a managed snapshot key and restored into the next run's fresh stack database (see Notes) | no | -| `PGDELTA_DEBUG` | bundled-engine debug artifacts | no | -| `SUPABASE_SERVICES_HOSTNAME` | local DB host for the bootstrap generate | no | -| `DOCKER_HOST` | tcp daemon host used as the local DB host fallback | no | -| `SUPABASE_USE_SLIM_IMAGES` | resolves the current-pin shadow Postgres and PG15+ realtime/storage/auth migrate-job images from the slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); majors 13/15 use `15.14.1.167` when the flag is on; historical pins, PG14, OrioleDB, and flag-off `15.8.1.085` stay on docker.io | no | +| Variable | Purpose | Required? | +| ---------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | +| `SUPABASE_HOME` | overrides the `~/.supabase` root used for the shadow baseline cache (and other CLI state) | no | +| `SUPABASE_SHADOW_CACHE` | shadow baseline cache; on by default, opt-out (`0`/`false`); the shadow's post-baseline state is saved under a managed snapshot key and restored into the next run's fresh stack database (see Notes) | no | +| `PGDELTA_DEBUG` | bundled-engine debug artifacts | no | +| `SUPABASE_SERVICES_HOSTNAME` | local DB host for the bootstrap generate | no | +| `DOCKER_HOST` | tcp daemon host used as the local DB host fallback | no | +| `SUPABASE_USE_SLIM_IMAGES` | resolves the current-pin shadow Postgres and PG15+ realtime/storage/auth migrate-job images from the slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); majors 13/15 use `15.14.1.167` when the flag is on; historical pins, PG14, OrioleDB tags the catalog does not pin, and flag-off `15.8.1.085` stay on docker.io | no | ## Exit Codes diff --git a/apps/cli/src/commands/db/shared/pgdelta-declarative-shadow-prep.ts b/apps/cli/src/commands/db/shared/pgdelta-declarative-shadow-prep.ts index 056d7c98b5..ce10681ed5 100644 --- a/apps/cli/src/commands/db/shared/pgdelta-declarative-shadow-prep.ts +++ b/apps/cli/src/commands/db/shared/pgdelta-declarative-shadow-prep.ts @@ -22,8 +22,17 @@ const DROP_IMAGE_DEFAULT_EXTENSION: Record<(typeof IMAGE_DEFAULT_EXTENSIONS)[num "uuid-ossp": 'DROP EXTENSION IF EXISTS "uuid-ossp"', }; +/** + * Image-installed extensions that stay in the shadow: tables already use their access method, so + * they cannot be dropped for replay and their declarations load as `IF NOT EXISTS`. + */ +const IMAGE_KEPT_EXTENSIONS = new Set(["orioledb"]); + const CREATE_EXTENSION_RE = - /\bCREATE\s+EXTENSION\s+(?:IF\s+NOT\s+EXISTS\s+)?(?:"([^"]+)"|([a-zA-Z_][\w$-]*))/gi; + /\b(CREATE\s+EXTENSION\s+)(IF\s+NOT\s+EXISTS\s+)?(?:"([^"]+)"|([a-zA-Z_][\w$-]*))/gi; + +const createExtensionName = (match: RegExpMatchArray): string => + (match[3] ?? match[4] ?? "").toLowerCase(); /** Blank comments and simple strings; keep offsets for locateSignature line mapping. */ export const maskSqlComments = (sql: string): string => @@ -37,7 +46,7 @@ export const declaredSqlExtensions = ( const declared = new Set(); for (const file of files) { for (const match of maskSqlComments(file.sql).matchAll(CREATE_EXTENSION_RE)) { - const name = (match[1] ?? match[2] ?? "").toLowerCase(); + const name = createExtensionName(match); if (name !== "") declared.add(name); } } @@ -77,14 +86,33 @@ const declarativeBaselinePrepStatements = ( return statements; }; -/** Recreate image pgjwt after a pgcrypto-only drop so omit still means keep. */ +const keepImageExtensionCreates = (sql: string): string => { + let kept = ""; + let cursor = 0; + for (const match of maskSqlComments(sql).matchAll(CREATE_EXTENSION_RE)) { + if (match[2] !== undefined || !IMAGE_KEPT_EXTENSIONS.has(createExtensionName(match))) continue; + const insertAt = match.index + (match[1] ?? "").length; + kept += `${sql.slice(cursor, insertAt)}IF NOT EXISTS `; + cursor = insertAt; + } + return cursor === 0 ? sql : kept + sql.slice(cursor); +}; + +/** + * Shadow-load view of the declarations: kept image extensions load idempotently, and image pgjwt + * is recreated after a pgcrypto-only drop so omit still means keep. + */ export const filesForDeclarativeShadowLoad = ( files: ReadonlyArray<{ readonly name: string; readonly sql: string }>, restorePgjwt: boolean, ): ReadonlyArray<{ readonly name: string; readonly sql: string }> => { - if (!restorePgjwt) return files; + const loaded = files.map((file) => { + const sql = keepImageExtensionCreates(file.sql); + return sql === file.sql ? file : { ...file, sql }; + }); + if (!restorePgjwt) return loaded; return [ - ...files, + ...loaded, { name: "_cli/restore-pgjwt.sql", sql: "CREATE EXTENSION IF NOT EXISTS pgjwt WITH SCHEMA extensions;\n", diff --git a/apps/cli/src/commands/db/shared/pgdelta-declarative-shadow-prep.unit.test.ts b/apps/cli/src/commands/db/shared/pgdelta-declarative-shadow-prep.unit.test.ts index a010a6fb81..42d33ec732 100644 --- a/apps/cli/src/commands/db/shared/pgdelta-declarative-shadow-prep.unit.test.ts +++ b/apps/cli/src/commands/db/shared/pgdelta-declarative-shadow-prep.unit.test.ts @@ -52,9 +52,42 @@ describe("filesForDeclarativeShadowLoad", () => { }, ]); }); + + it("loads an image-installed orioledb declaration idempotently and leaves other SQL as written", () => { + const orioledb = { + name: "_cluster/extensions/orioledb.sql", + sql: '-- CREATE EXTENSION orioledb;\nCREATE EXTENSION "orioledb" SCHEMA "extensions";\n\nCOMMENT ON EXTENSION "orioledb" IS \'OrioleDB\';\n', + }; + const alreadyIdempotent = { + name: "public/01.sql", + sql: "create extension if not exists orioledb;\nCREATE EXTENSION pgcrypto;", + }; + expect(filesForDeclarativeShadowLoad([orioledb, alreadyIdempotent], false)).toEqual([ + { + name: orioledb.name, + sql: '-- CREATE EXTENSION orioledb;\nCREATE EXTENSION IF NOT EXISTS "orioledb" SCHEMA "extensions";\n\nCOMMENT ON EXTENSION "orioledb" IS \'OrioleDB\';\n', + }, + alreadyIdempotent, + ]); + }); }); describe("prepareDeclarativeShadow", () => { + it.live("keeps an image-installed orioledb instead of dropping it", () => { + const queries: string[] = []; + const client = fakeShadowClient((sql) => { + queries.push(sql); + return Promise.resolve({ rows: [] }); + }); + return Effect.gen(function* () { + const prep = yield* prepareDeclarativeShadow(client, [ + { name: "_cluster/extensions/orioledb.sql", sql: 'CREATE EXTENSION "orioledb";' }, + ]); + expect(prep.restorePgjwt).toBe(false); + expect(queries).toEqual([]); + }); + }); + it.live("skips the shadow when declarations omit image-default extensions", () => { const queries: string[] = []; const client = fakeShadowClient((sql) => { diff --git a/apps/cli/src/commands/db/start/SIDE_EFFECTS.md b/apps/cli/src/commands/db/start/SIDE_EFFECTS.md index c0f7385151..812b33ce65 100644 --- a/apps/cli/src/commands/db/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/db/start/SIDE_EFFECTS.md @@ -40,7 +40,7 @@ composition reuses too — see that command's `SIDE_EFFECTS.md`): `SUPABASE_USE_SLIM_IMAGES` rewrites the current Dockerfile pin (and majors 13/15's published slim PG15 pin, `15.14.1.167`) to `ghcr.io/supabase/cli/postgres`; a historical `.temp/postgres-version` pin, - PG14, OrioleDB, and flag-off majors 13/15 (`15.8.1.085`) stay on docker.io. + PG14, OrioleDB tags the catalog does not pin, and flag-off majors 13/15 (`15.8.1.085`) stay on docker.io. The restore entrypoint is the same on both families. 6. Wait for the container to become healthy (`db.health_timeout`, default `2m`). A timeout fails the command UNLESS `--from-backup` is set, in which case it is swallowed (a large @@ -123,25 +123,25 @@ API request over the active context's local unix socket / named pipe: ## Environment Variables -| Variable | Purpose | Required? | -| -------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | -| `SUPABASE_PROJECT_ID` | overrides the local container id | no | -| `SUPABASE_DB_PORT` | overrides `db.port` (the published host port) | no | -| `SUPABASE_DB_MAJOR_VERSION` | overrides `db.major_version` (image selection, schema branch) | no | -| `SUPABASE_DB_HEALTH_TIMEOUT` | overrides `db.health_timeout` | no | -| `SUPABASE_DB_SETTINGS_*` | overrides individual `[db.settings]` fields | no | -| `SUPABASE_DB_ORIOLEDB_VERSION` | overrides `db.orioledb_version` (image + env) | no | -| `SUPABASE_EXPERIMENTAL_S3_{HOST,REGION,ACCESS_KEY,SECRET_KEY}` | OrioleDB S3 env overrides | no | -| `SUPABASE_REALTIME_ENABLED` | gates the fresh-volume realtime migrate job | no | -| `SUPABASE_REALTIME_IP_VERSION` / `_MAX_HEADER_LENGTH` | realtime migrate job env overrides | no | -| `SUPABASE_STORAGE_ENABLED` | gates the fresh-volume storage migrate job | no | -| `SUPABASE_STORAGE_FILE_SIZE_LIMIT` | storage migrate job env override | no | -| `SUPABASE_AUTH_ENABLED` | gates the fresh-volume auth migrate job | no | -| `SUPABASE_AUTH_EXTERNAL_URL` / `SUPABASE_AUTH_SITE_URL` | auth migrate job env overrides | no | -| `SUPABASE_AUTH_JWT_EXPIRY` | Postgres's `JWT_EXP` env / signing | no | -| `SUPABASE_EXPERIMENTAL` (or `--experimental`) | fresh volume + no pg-delta: applies `db.migrations.schema_paths` files instead of `migrations/*.sql` | no | -| `DOCKER_HOST` / `DOCKER_CONTEXT` / `DOCKER_TLS_VERIFY` / `DOCKER_CERT_PATH` / `DOCKER_API_VERSION` / `DOCKER_CONFIG` | Read from the ambient shell environment to pick the Docker daemon this whole command talks to (project dotenv files deliberately never override Docker client keys) | no | -| `SUPABASE_USE_SLIM_IMAGES` | resolves the current Dockerfile pin (and majors 13/15's published slim PG15 pin, `15.14.1.167`) and PG15+ realtime/storage/auth migrate-job images from the slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); historical `.temp` pins, PG14, OrioleDB, and flag-off majors 13/15 (`15.8.1.085`) stay on docker.io | no | +| Variable | Purpose | Required? | +| -------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | +| `SUPABASE_PROJECT_ID` | overrides the local container id | no | +| `SUPABASE_DB_PORT` | overrides `db.port` (the published host port) | no | +| `SUPABASE_DB_MAJOR_VERSION` | overrides `db.major_version` (image selection, schema branch) | no | +| `SUPABASE_DB_HEALTH_TIMEOUT` | overrides `db.health_timeout` | no | +| `SUPABASE_DB_SETTINGS_*` | overrides individual `[db.settings]` fields | no | +| `SUPABASE_DB_ORIOLEDB_VERSION` | overrides `db.orioledb_version` (image + env) | no | +| `SUPABASE_EXPERIMENTAL_S3_{HOST,REGION,ACCESS_KEY,SECRET_KEY}` | OrioleDB S3 env overrides | no | +| `SUPABASE_REALTIME_ENABLED` | gates the fresh-volume realtime migrate job | no | +| `SUPABASE_REALTIME_IP_VERSION` / `_MAX_HEADER_LENGTH` | realtime migrate job env overrides | no | +| `SUPABASE_STORAGE_ENABLED` | gates the fresh-volume storage migrate job | no | +| `SUPABASE_STORAGE_FILE_SIZE_LIMIT` | storage migrate job env override | no | +| `SUPABASE_AUTH_ENABLED` | gates the fresh-volume auth migrate job | no | +| `SUPABASE_AUTH_EXTERNAL_URL` / `SUPABASE_AUTH_SITE_URL` | auth migrate job env overrides | no | +| `SUPABASE_AUTH_JWT_EXPIRY` | Postgres's `JWT_EXP` env / signing | no | +| `SUPABASE_EXPERIMENTAL` (or `--experimental`) | fresh volume + no pg-delta: applies `db.migrations.schema_paths` files instead of `migrations/*.sql` | no | +| `DOCKER_HOST` / `DOCKER_CONTEXT` / `DOCKER_TLS_VERIFY` / `DOCKER_CERT_PATH` / `DOCKER_API_VERSION` / `DOCKER_CONFIG` | Read from the ambient shell environment to pick the Docker daemon this whole command talks to (project dotenv files deliberately never override Docker client keys) | no | +| `SUPABASE_USE_SLIM_IMAGES` | resolves the current Dockerfile pin (and majors 13/15's published slim PG15 pin, `15.14.1.167`) and PG15+ realtime/storage/auth migrate-job images from the slim `ghcr.io/supabase/cli` builds (`true`/`1` enable); historical `.temp` pins, PG14, OrioleDB tags the catalog does not pin, and flag-off majors 13/15 (`15.8.1.085`) stay on docker.io | no | `--network-id` (a global CLI flag, not an environment variable — `command-internal/global-flags.ts`) forces every created container/network onto that Docker network instead of the generated diff --git a/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts index 45a41ae98b..a7c6507b4a 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts @@ -134,14 +134,17 @@ enabled = false }), ); - it.live("rejects the existing OrioleDB environment override", () => + it.live("applies the OrioleDB environment override before checking the catalog", () => Effect.gen(function* () { const root = yield* project('project_id = "stack-config-env-orioledb"\n'); - const exit = yield* withEnvVar("SUPABASE_DB_ORIOLEDB_VERSION", "15.1.1.14", load(root)).pipe( + const exit = yield* withEnvVar("SUPABASE_DB_ORIOLEDB_VERSION", "17.0.0.000", load(root)).pipe( Effect.exit, ); expect(Exit.isFailure(exit)).toBe(true); - if (Exit.isFailure(exit)) expect(String(exit.cause)).toContain("db.orioledb_version"); + if (Exit.isFailure(exit)) + expect(String(exit.cause)).toContain( + "db.orioledb_version = 17.0.0.000 requires a published OrioleDB artifact", + ); }), ); }); diff --git a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts index bec7168a05..ab90fddb91 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts @@ -65,6 +65,7 @@ enabled = true expect(keys.gotrueJwtKeys).toBeUndefined(); expect(keys.publicSigningKeys).toBeUndefined(); const database = recipes.get("database"); + expect(database?.service === "database" ? database.config.version : undefined).toBe("17"); expect( database?.service === "database" ? database.config.rootKey : undefined, ).toBeUndefined(); @@ -343,35 +344,38 @@ s3_secret_key = "env(S3_SECRET_KEY)" }).pipe(Effect.provide(BunServices.layer)); }); - it.live("rejects OrioleDB and ignores its inactive S3 settings", () => - Effect.gen(function* () { - const orioledb = yield* project(`project_id = "stack-config-orioledb" + it.live( + "fails closed on an unpublished OrioleDB version and ignores its inactive S3 settings", + () => + Effect.gen(function* () { + const unpublished = + "db.orioledb_version = 17.0.0.000 requires a published OrioleDB artifact; supported OrioleDB versions:"; + const orioledb = yield* project(`project_id = "stack-config-orioledb" [experimental] -orioledb_version = "15.1.1.14" +orioledb_version = "17.0.0.000" `); - const orioledbExit = yield* load(orioledb).pipe(Effect.exit); - expect(Exit.isFailure(orioledbExit)).toBe(true); - if (Exit.isFailure(orioledbExit)) - expect(String(orioledbExit.cause)).toContain("db.orioledb_version"); + const orioledbExit = yield* load(orioledb).pipe(Effect.exit); + expect(Exit.isFailure(orioledbExit)).toBe(true); + if (Exit.isFailure(orioledbExit)) expect(String(orioledbExit.cause)).toContain(unpublished); - // The same rejection applies to the canonical `[db]` location, not just the deprecated - // `[experimental]` alias. - const orioledbCanonical = yield* project(`project_id = "stack-config-orioledb-db" + // The same check applies to the canonical `[db]` location, not just the deprecated + // `[experimental]` alias. + const orioledbCanonical = yield* project(`project_id = "stack-config-orioledb-db" [db] -orioledb_version = "15.1.1.14" +orioledb_version = "17.0.0.000" `); - const orioledbCanonicalExit = yield* load(orioledbCanonical).pipe(Effect.exit); - expect(Exit.isFailure(orioledbCanonicalExit)).toBe(true); - if (Exit.isFailure(orioledbCanonicalExit)) - expect(String(orioledbCanonicalExit.cause)).toContain("db.orioledb_version"); + const orioledbCanonicalExit = yield* load(orioledbCanonical).pipe(Effect.exit); + expect(Exit.isFailure(orioledbCanonicalExit)).toBe(true); + if (Exit.isFailure(orioledbCanonicalExit)) + expect(String(orioledbCanonicalExit.cause)).toContain(unpublished); - const s3 = yield* project(`project_id = "stack-config-experimental-s3" + const s3 = yield* project(`project_id = "stack-config-experimental-s3" [experimental] s3_host = "s3.example.test" `); - const s3Config = yield* load(s3); - expect(s3Config.source.experimental.s3_host).toBe("s3.example.test"); - }).pipe(Effect.provide(BunServices.layer)), + const s3Config = yield* load(s3); + expect(s3Config.source.experimental.s3_host).toBe("s3.example.test"); + }).pipe(Effect.provide(BunServices.layer)), ); it.live("records OrioleDB selection on the command event before rejecting it", () => diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 4c6e60d465..3e9a40899b 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -31,6 +31,10 @@ credentials when present. Starting with Studio creates `supabase/snippets/`, whe snippets. Email template `content_path` values and third-party identity providers remain unsupported: they require template serving and shared external JWKS verification respectively. +`db.orioledb_version` (or `SUPABASE_DB_ORIOLEDB_VERSION`) runs the catalog's OrioleDB build of that +version instead of the stock major. Loading fails unless the catalog pins that OrioleDB version and +`db.major_version` matches its major; the error lists the pinned OrioleDB versions. The +`experimental.s3_*` OrioleDB settings are not forwarded. Secrets needed by enabled services are passed to the runtime. State and service data live under `$SUPABASE_HOME/stacks//` (`~/.supabase/stacks//` by default); native artifacts @@ -91,6 +95,12 @@ project configuration file is unchanged. A changed endpoint, artifact version, o version fails before modifying the stopped composition, naming the `config.toml` key or `SUPABASE_*` env var behind the change with its saved and requested values, and suggesting either reverting it or running the stack's exact `supabase stack destroy` command to recreate it. +Switching between stock PostgreSQL and OrioleDB is an artifact version change, reported as +`db.orioledb_version` (or `SUPABASE_DB_ORIOLEDB_VERSION`) with `unset` standing for stock. +Initialized database data is reused only on its own release line (major plus stock or OrioleDB). A +first start records the requested line before initializing data, so a first start interrupted +before readiness resumes on that line; unmarked data without a recorded line can prove only its +major, so it is never reused for OrioleDB. ## First startup and retries @@ -140,8 +150,9 @@ example. Failures retain typed command errors and package diagnostics. Telemetry after success or failure. A rejected configuration change additionally carries `stack_changes` on the JSON/stream-json error -envelope: one entry per affected service (a shared setting such as the API port appears once per -API-backed service, unlike the deduplicated text message), each with `service`, `path` (the +envelope: one entry per affected service and setting (a shared setting such as the API port appears +once per API-backed service, unlike the deduplicated text message; a database version change that +moves both `db.major_version` and `db.orioledb_version` has an entry for each), each with `service`, `path` (the composition planner's dotted path, e.g. `endpoints.http.port`, not a `config.toml` key), `key`, `saved`, `requested`, and `editable`. `recreate_command` is the exact `supabase stack destroy --stack-id ` invocation, without `--yes`, since destroy deletes local database data; running it diff --git a/apps/cli/src/commands/experimental/stack/start/start.handler.ts b/apps/cli/src/commands/experimental/stack/start/start.handler.ts index 80ba093d48..13885596fa 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.handler.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.handler.ts @@ -1,4 +1,8 @@ -import { defaultRuntime, postgresVersion } from "@supabase/stack/internal/artifacts"; +import { + defaultRuntime, + isOrioledbVersion, + postgresVersion, +} from "@supabase/stack/internal/artifacts"; import { connectionEnv, renderStackSummary, @@ -52,6 +56,7 @@ import { loadStackConfig, stackEndpointSetting, stackMajorVersionSetting, + stackOrioledbVersionSetting, type StackEndpointSetting, } from "../../../../command-internal/stack-config.ts"; import { envOverride } from "../../../../command-internal/local-config-values.ts"; @@ -186,6 +191,10 @@ const databaseVersionOf = ( const majorVersionOf = (version: string): string => version.split(".")[0] ?? version; +/** The `db.orioledb_version` value behind a database artifact version, or `unset` for stock. */ +const orioledbVersionOf = (version: string): string => + isOrioledbVersion(version) ? version.replace(/-orioledb$/u, "") : "unset"; + /** Renders a dotted config key as its `config.toml` section/key pair, e.g. `[db] major_version`. */ const formatConfigPath = (path: string): string => { const segments = path.split("."); @@ -203,8 +212,8 @@ const settingKeyLabel = ( : formatConfigPath(setting.configPath); /** - * One incompatible path, reported as the JSON/stream-json error envelope's `stack_changes` - * entries (contract documented in `SIDE_EFFECTS.md`). `editable` marks whether `key` is a + * One setting behind an incompatible path, reported as the JSON/stream-json error envelope's + * `stack_changes` entries (contract documented in `SIDE_EFFECTS.md`). `editable` marks whether `key` is a * `config.toml` key or env var the user can revert, or plain wording for a catalog-pinned * artifact or Postgres build. */ @@ -217,13 +226,13 @@ interface StructuredSettingChange { readonly editable: boolean; } -const describeSettingChange = ( +const describeSettingChanges = ( service: PlannedInstance["service"], path: string, savedCreation: ServiceCreation | undefined, requestedCreation: ServiceCreationInput | undefined, projectEnvValues: Readonly>, -): StructuredSettingChange => { +): ReadonlyArray => { if (service === "database" && path === "config.version") { // `postgresVersion` resolves a bare major alias (e.g. "17") to the pinned build the // composition plan actually compared, so the saved/requested pair reflects what changed. @@ -231,48 +240,68 @@ const describeSettingChange = ( const requestedVersion = postgresVersion(databaseVersionOf(requestedCreation) ?? "unknown"); const savedMajor = majorVersionOf(savedVersion); const requestedMajor = majorVersionOf(requestedVersion); - // Same major but different pinned build: `major_version` doesn't control this, so reverting - // it wouldn't fix anything — name the actual (unpinnable) versions instead. - if (savedMajor === requestedMajor) - return { + const savedOrioledb = orioledbVersionOf(savedVersion); + const requestedOrioledb = orioledbVersionOf(requestedVersion); + const changes: Array = []; + if (savedMajor !== requestedMajor) + changes.push({ service, path, - key: "Postgres build", - saved: savedVersion, - requested: requestedVersion, - editable: false, - }; - return { - service, - path, - key: settingKeyLabel(stackMajorVersionSetting, projectEnvValues), - saved: savedMajor, - requested: requestedMajor, - editable: true, - }; + key: settingKeyLabel(stackMajorVersionSetting, projectEnvValues), + saved: savedMajor, + requested: requestedMajor, + editable: true, + }); + if (savedOrioledb !== requestedOrioledb) + changes.push({ + service, + path, + key: settingKeyLabel(stackOrioledbVersionSetting, projectEnvValues), + saved: savedOrioledb, + requested: requestedOrioledb, + editable: true, + }); + // Same major and engine but a different pinned build: no setting controls this, so name the + // actual (unpinnable) versions instead. + return changes.length > 0 + ? changes + : [ + { + service, + path, + key: "Postgres build", + saved: savedVersion, + requested: requestedVersion, + editable: false, + }, + ]; } const endpointName = path.startsWith("endpoints.") ? path.split(".")[1] : undefined; const setting = endpointName === undefined ? undefined : stackEndpointSetting(service, endpointName); if (endpointName !== undefined && setting !== undefined) - return { - service, - path, - key: settingKeyLabel(setting, projectEnvValues), - saved: endpointPortLabel(savedCreation?.endpoints, endpointName), - requested: endpointPortLabel(requestedCreation?.endpoints, endpointName), - editable: true, - }; + return [ + { + service, + path, + key: settingKeyLabel(setting, projectEnvValues), + saved: endpointPortLabel(savedCreation?.endpoints, endpointName), + requested: endpointPortLabel(requestedCreation?.endpoints, endpointName), + editable: true, + }, + ]; // No config.toml key or env var covers this path (e.g. the catalog-pinned artifact `version`): // name it plainly instead of implying a setting the user could edit. - return { - service, - path, - key: path === "version" ? `${service} artifact version` : `${service} ${path}`, - saved: path === "version" ? (savedCreation?.version ?? "unknown") : "changed", - requested: path === "version" ? (requestedCreation?.version ?? "unknown") : "changed", - editable: false, - }; + return [ + { + service, + path, + key: path === "version" ? `${service} artifact version` : `${service} ${path}`, + saved: path === "version" ? (savedCreation?.version ?? "unknown") : "changed", + requested: path === "version" ? (requestedCreation?.version ?? "unknown") : "changed", + editable: false, + }, + ]; }; /** Every incompatible path across every rejected saved member, as one structured list. */ @@ -287,8 +316,8 @@ const incompatibleSettingChanges = ( .flatMap((entry) => // Narrowed by the filter above; `Extract` isn't inferred through `.filter`. entry.change === "incompatible" - ? entry.paths.map((path) => - describeSettingChange( + ? entry.paths.flatMap((path) => + describeSettingChanges( entry.service, path, savedConfigById.get(entry.id), diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index ab3f102e4c..5b57552d9c 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -1484,6 +1484,71 @@ describe("experimental stack start", () => { }).pipe(Effect.provide(BunServices.layer)), ); + for (const target of [ + { + name: "[db] orioledb_version when a saved stock stack switches to OrioleDB", + before: "", + after: '[db]\norioledb_version = "17.11.0.002"\n', + env: undefined, + changes: "[db] orioledb_version: saved unset, requested 17.11.0.002", + revert: "Revert [db] orioledb_version to its saved value", + }, + { + name: "[db] orioledb_version when a saved OrioleDB stack switches back to stock", + before: '[experimental]\norioledb_version = "17.11.0.002"\n', + after: "", + env: undefined, + changes: "[db] orioledb_version: saved 17.11.0.002, requested unset", + revert: "Revert [db] orioledb_version to its saved value", + }, + { + name: "both [db] major_version and [db] orioledb_version when stock 15 becomes OrioleDB 17", + before: "[db]\nmajor_version = 15\n", + after: '[db]\norioledb_version = "17.11.0.002"\n', + env: undefined, + changes: + "[db] major_version: saved 15, requested 17; [db] orioledb_version: saved unset, requested 17.11.0.002", + revert: "Revert the settings listed to their saved values", + }, + { + name: "SUPABASE_DB_ORIOLEDB_VERSION when it switches a saved stock stack to OrioleDB", + before: "", + after: "", + env: "17.11.0.002", + changes: "SUPABASE_DB_ORIOLEDB_VERSION: saved unset, requested 17.11.0.002", + revert: "Revert SUPABASE_DB_ORIOLEDB_VERSION to its saved value", + }, + ]) + it.live(`names ${target.name}`, () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-start-orioledb-" }); + yield* fs.makeDirectory(`${root}/supabase`, { recursive: true }); + const configPath = `${root}/supabase/config.toml`; + yield* fs.writeFileString(configPath, `project_id = "orioledb"\n${target.before}`); + const fixture = fakeStack(); + yield* stackStart(flags()).pipe(Effect.provide(layers(root, fixture))); + + yield* fs.writeFileString(configPath, `project_id = "orioledb"\n${target.after}`); + yield* fixture.stack.composition.stop; + const restart = stackStart(flags()).pipe( + Effect.provide(layers(root, fixture)), + Effect.flip, + ); + const error = yield* target.env === undefined + ? restart + : withEnvVar("SUPABASE_DB_ORIOLEDB_VERSION", target.env, restart); + + expect(error).toMatchObject({ + reason: "invalid-config", + message: `The saved stack cannot adopt these changes: ${target.changes}`, + suggestion: expect.stringContaining( + `${target.revert} to keep the stack and its data, or run \`supabase stack destroy --stack-id ${fixture.stack.id}\``, + ), + }); + }).pipe(Effect.provide(BunServices.layer)), + ); + it.live( "drops the revert sentence for an artifact-version-only mismatch and explains the fix in plain language", () => diff --git a/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md b/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md index 6c7a10d777..aa6ecff6c3 100644 --- a/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/migration/squash/SIDE_EFFECTS.md @@ -86,7 +86,7 @@ live shadow available. `SUPABASE_YES`, `DB_PASSWORD`, `SUPABASE_ACCESS_TOKEN`, `SUPABASE_SERVICES_HOSTNAME`, `DOCKER_HOST`/`DOCKER_CONTEXT`/`DOCKER_CONFIG`, `SUPABASE_NETWORK_ID`, -`SUPABASE_INTERNAL_IMAGE_REGISTRY`, `SUPABASE_USE_SLIM_IMAGES` (current-pin shadow Postgres and PG15+ realtime/storage/auth migrate-job images → slim `ghcr.io/supabase/cli`; historical pins, PG14, OrioleDB, flag-off `15.8.1.085` stay on docker.io), `SUPABASE_PROJECT_ID`, `SUPABASE_DEBUG`, +`SUPABASE_INTERNAL_IMAGE_REGISTRY`, `SUPABASE_USE_SLIM_IMAGES` (current-pin shadow Postgres and PG15+ realtime/storage/auth migrate-job images → slim `ghcr.io/supabase/cli`; historical pins, PG14, OrioleDB tags the catalog does not pin, flag-off `15.8.1.085` stay on docker.io), `SUPABASE_PROJECT_ID`, `SUPABASE_DEBUG`, `SUPABASE_EXPERIMENTAL`, `SUPABASE_SHADOW_CACHE` (stack shadow baseline cache; on by default, falsy disables restore and publication). ## Exit Codes diff --git a/apps/cli/src/commands/services/services-local-stack.ts b/apps/cli/src/commands/services/services-local-stack.ts index 4658efcdaf..839990a846 100644 --- a/apps/cli/src/commands/services/services-local-stack.ts +++ b/apps/cli/src/commands/services/services-local-stack.ts @@ -5,7 +5,11 @@ import { } from "@supabase/stack/internal/artifacts"; import { Effect, Result } from "effect"; import { loadLocalProjectContext } from "../../command-internal/local-project-context.ts"; -import { envOverrideMajorVersion } from "../../command-internal/local-config-values.ts"; +import { + envOverride, + envOverrideMajorVersion, +} from "../../command-internal/local-config-values.ts"; +import { stackDatabaseVersion } from "../../command-internal/stack-database-version.ts"; import { upstreamVersionFromTag } from "../../shared/services/services.shared.ts"; import type { ServiceVersionRow } from "../../shared/services/services.shared.ts"; import type { RemoteServiceName } from "../../shared/services/services.shared.ts"; @@ -23,31 +27,36 @@ export const stackServiceVersions = Effect.fn("services.stackServiceVersions")(f ) { const context = yield* loadLocalProjectContext(workdir, (message) => message).pipe(Effect.result); let configError: string | undefined; - let major: number | undefined; + let databaseVersion: string | undefined; if (Result.isFailure(context)) configError = context.failure; else { - const resolvedMajor = yield* Effect.try({ + const { config, projectEnvValues } = context.success; + const resolved = yield* Effect.try({ try: () => { - const value = envOverrideMajorVersion( - context.success.config.db.major_version, - context.success.projectEnvValues, - ); + const value = envOverrideMajorVersion(config.db.major_version, projectEnvValues); if (value !== 15 && value !== 17) throw new Error(`unsupported PostgreSQL major version: ${value}`); - return value; + return stackDatabaseVersion({ + major_version: value, + orioledb_version: envOverride( + "SUPABASE_DB_ORIOLEDB_VERSION", + config.db.orioledb_version, + projectEnvValues, + ), + }); }, catch: (cause) => (cause instanceof Error ? cause.message : String(cause)), - }).pipe(Effect.result); - if (Result.isFailure(resolvedMajor)) configError = resolvedMajor.failure; - else major = resolvedMajor.success; + }).pipe(Effect.flatMap(Effect.fromResult), Effect.result); + if (Result.isFailure(resolved)) configError = resolved.failure; + else databaseVersion = resolved.success; } yield* Effect.annotateCurrentSpan({ "config.load_failed": configError !== undefined }); return yield* Effect.forEach(artifactServiceKinds(), (service) => Effect.gen(function* () { const artifact = yield* resolveArtifact({ service, - ...(service === "database" && major !== undefined - ? { version: postgresVersion(String(major)) } + ...(service === "database" && databaseVersion !== undefined + ? { version: postgresVersion(databaseVersion) } : {}), }); const name = artifact.image.split("@")[0]?.replace(/:[^/:]+$/, "") ?? artifact.image; diff --git a/apps/cli/src/commands/start/SIDE_EFFECTS.md b/apps/cli/src/commands/start/SIDE_EFFECTS.md index fc600a2b31..c3d9c34c6c 100644 --- a/apps/cli/src/commands/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/start/SIDE_EFFECTS.md @@ -193,19 +193,19 @@ not implemented. ## Environment Variables -| Variable | Purpose | Required? | -| -------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | -| `SUPABASE_*` (any dotted config field) | Generic Viper-style `AutomaticEnv` override of any `config.toml` field (e.g. `SUPABASE_AUTH_ENABLED`, `SUPABASE_API_PORT`) | no | -| `SUPABASE_EXPERIMENTAL` (or `--experimental`) | Fresh volume + no pg-delta: applies `db.migrations.schema_paths` files instead of `migrations/*.sql` (see "Fresh-volume DB setup" above) | no | -| `SUPABASE_INTERNAL_IMAGE_REGISTRY` | Overrides the image registry used to resolve every service's image | no | -| `SUPABASE_PROJECT_ID` | Overrides the resolved local project id (env → config.toml → workdir basename) | no | -| `SUPABASE_WORKDIR` | Resolves `CommandSettings.workdir` | no | -| `SUPABASE_YES` (or `--yes`) | Auto-confirms the fresh-volume bucket-seed overwrite/prune prompts (shell or project dotenv, same as `seed buckets`) | no | -| `BITBUCKET_CLONE_DIR` | When non-empty, drops named volumes and `--security-opt` from every container create | no | -| `DOCKER_HOST` / `DOCKER_CONTEXT` / `DOCKER_TLS_VERIFY` / `DOCKER_CERT_PATH` / `DOCKER_API_VERSION` / `DOCKER_CONFIG` | Read (ambient shell OR a project `.env`/`.env.`/`.env.local` file) to discover the Docker daemon this whole command talks to; `DOCKER_HOST` is also re-derived and set on Vector's container env so it can reach the host's Docker socket for log collection | no | -| `KONG_NGINX_WORKER_PROCESSES` | Read (ambient shell or project dotenv) into Kong's own container env (defaults to `"1"` when unset) | no | -| `HTTP_PROXY` / `http_proxy` / `HTTPS_PROXY` / `https_proxy` / `NO_PROXY` / `no_proxy` | Bun proxy settings. After project dotenv and container creation, `start` appends `localhost,127.0.0.1,[::1]` to the effective no-proxy value before local Kong probes and seeding; it never changes project/container env and ends with this CLI process. | no | -| `SUPABASE_USE_SLIM_IMAGES` | Ambient `process.env` only (`true`/`1` enable) — not project dotenv. Rewrites current Dockerfile pins to `ghcr.io/supabase/cli/` (including PG15+ realtime/storage/auth migrate jobs). Kong, PG14, OrioleDB, historical Postgres pins, and `deno_version = 1` remain non-slim (still subject to `SUPABASE_INTERNAL_IMAGE_REGISTRY`). Majors 13/15 use the published slim PG15 pin (`15.14.1.167`) when the flag is on; flag-off keeps `15.8.1.085`. `SUPABASE_INTERNAL_IMAGE_REGISTRY` does not apply to slim refs | no | +| Variable | Purpose | Required? | +| -------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- | +| `SUPABASE_*` (any dotted config field) | Generic Viper-style `AutomaticEnv` override of any `config.toml` field (e.g. `SUPABASE_AUTH_ENABLED`, `SUPABASE_API_PORT`) | no | +| `SUPABASE_EXPERIMENTAL` (or `--experimental`) | Fresh volume + no pg-delta: applies `db.migrations.schema_paths` files instead of `migrations/*.sql` (see "Fresh-volume DB setup" above) | no | +| `SUPABASE_INTERNAL_IMAGE_REGISTRY` | Overrides the image registry used to resolve every service's image | no | +| `SUPABASE_PROJECT_ID` | Overrides the resolved local project id (env → config.toml → workdir basename) | no | +| `SUPABASE_WORKDIR` | Resolves `CommandSettings.workdir` | no | +| `SUPABASE_YES` (or `--yes`) | Auto-confirms the fresh-volume bucket-seed overwrite/prune prompts (shell or project dotenv, same as `seed buckets`) | no | +| `BITBUCKET_CLONE_DIR` | When non-empty, drops named volumes and `--security-opt` from every container create | no | +| `DOCKER_HOST` / `DOCKER_CONTEXT` / `DOCKER_TLS_VERIFY` / `DOCKER_CERT_PATH` / `DOCKER_API_VERSION` / `DOCKER_CONFIG` | Read (ambient shell OR a project `.env`/`.env.`/`.env.local` file) to discover the Docker daemon this whole command talks to; `DOCKER_HOST` is also re-derived and set on Vector's container env so it can reach the host's Docker socket for log collection | no | +| `KONG_NGINX_WORKER_PROCESSES` | Read (ambient shell or project dotenv) into Kong's own container env (defaults to `"1"` when unset) | no | +| `HTTP_PROXY` / `http_proxy` / `HTTPS_PROXY` / `https_proxy` / `NO_PROXY` / `no_proxy` | Bun proxy settings. After project dotenv and container creation, `start` appends `localhost,127.0.0.1,[::1]` to the effective no-proxy value before local Kong probes and seeding; it never changes project/container env and ends with this CLI process. | no | +| `SUPABASE_USE_SLIM_IMAGES` | Ambient `process.env` only (`true`/`1` enable) — not project dotenv. Rewrites current Dockerfile pins to `ghcr.io/supabase/cli/` (including PG15+ realtime/storage/auth migrate jobs). Kong, PG14, OrioleDB tags the catalog does not pin, historical Postgres pins, and `deno_version = 1` remain non-slim (still subject to `SUPABASE_INTERNAL_IMAGE_REGISTRY`). Majors 13/15 use the published slim PG15 pin (`15.14.1.167`) when the flag is on; flag-off keeps `15.8.1.085`. `SUPABASE_INTERNAL_IMAGE_REGISTRY` does not apply to slim refs | no | `docker`/`podman` must be resolvable on `PATH` — same fallback behavior as `stop`/`status`. diff --git a/apps/cli/src/shared/services/slim-images.ts b/apps/cli/src/shared/services/slim-images.ts index 503114e438..1683989549 100644 --- a/apps/cli/src/shared/services/slim-images.ts +++ b/apps/cli/src/shared/services/slim-images.ts @@ -60,20 +60,11 @@ export interface SlimCatalogPin { readonly version: string; } -/** OrioleDB tags are docker.io-only; slim-services does not publish them. */ -function isOrioleImage(image: string): boolean { - const tag = imageTag(image); - return tag !== undefined && tag.toLowerCase().includes("orioledb"); -} - /** * Slim service and tag for a Dockerfile alias. Absent when that alias has no - * slim build (kong, the one-shot job images, and OrioleDB tags). + * slim build (kong and the one-shot job images). */ export function slimCatalogPin(alias: string, image: string): SlimCatalogPin | undefined { - if (isOrioleImage(image)) { - return undefined; - } const service = SLIM_SERVICE_LOOKUP[alias]; if (service === undefined) { return undefined; diff --git a/apps/cli/src/shared/services/slim-images.unit.test.ts b/apps/cli/src/shared/services/slim-images.unit.test.ts index 998611c15b..a5d14f083f 100644 --- a/apps/cli/src/shared/services/slim-images.unit.test.ts +++ b/apps/cli/src/shared/services/slim-images.unit.test.ts @@ -14,8 +14,9 @@ import { usesSlimImageRuntime, } from "./slim-images.ts"; -// Only `auth` is pinned, so every other alias falls through to its upstream image. `vi.mock` -// factories are hoisted above top-level statements, so the fixture is inlined. +// Only `auth` and one OrioleDB postgres build are pinned, so every other alias falls through to +// its upstream image. `vi.mock` factories are hoisted above top-level statements, so the fixture +// is inlined. vi.mock("@supabase/stack/internal/artifacts", () => { const digest = "d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c"; const nativePin = { archive: digest, manifest: digest }; @@ -35,12 +36,28 @@ vi.mock("@supabase/stack/internal/artifacts", () => { }, }, }, + { + service: "database", + sourceService: "postgres", + pin: { + upstreamVersion: "17.11.0.002-orioledb", + revision: 0, + image: `ghcr.io/supabase/cli/postgres:17.11.0.002-orioledb-r0@sha256:${digest}`, + natives: { + "darwin-arm64": nativePin, + "linux-amd64": nativePin, + "linux-arm64": nativePin, + }, + }, + }, ], }; }); const AUTH_FIXTURE_PIN_IMAGE = "ghcr.io/supabase/cli/auth:v2.197.0-r0@sha256:d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c"; +const ORIOLEDB_FIXTURE_PIN_IMAGE = + "ghcr.io/supabase/cli/postgres:17.11.0.002-orioledb-r0@sha256:d348483ad1141c54bfb4eaae801f5385fe1c2970fc106f95f531b5247092d52c"; afterEach(() => { vi.unstubAllEnvs(); @@ -108,9 +125,11 @@ describe("slimCatalogPin", () => { }); }); - it("excludes OrioleDB tags", () => { - expect(slimCatalogPin("pg", "supabase/postgres:16.0.0.1-orioledb")).toBeUndefined(); - expect(slimCatalogPin("pg", "supabase/postgres:orioledb-15.1.0.55")).toBeUndefined(); + it("keeps an OrioleDB tag whole, so it matches only an OrioleDB catalog pin", () => { + expect(slimCatalogPin("pg", "supabase/postgres:17.11.0.002-orioledb")).toEqual({ + service: "postgres", + version: "17.11.0.002-orioledb", + }); }); it("strips vector's docker.io -alpine variant suffix", () => { @@ -147,12 +166,20 @@ describe("toSlimImage", () => { it("returns undefined, keeping the upstream image, when the tag isn't in the catalog", () => { expect(toSlimImage("gotrue", "supabase/gotrue:v2.100.0")).toBeUndefined(); - // `pg` has no entry at all in this fixture catalog. + // `pg` has only an OrioleDB entry in this fixture catalog. expect(toSlimImage("pg", "supabase/postgres:17.6.1.164")).toBeUndefined(); }); - it("returns undefined for aliases and tags slimCatalogPin already excludes", () => { + it("maps a pinned OrioleDB tag to its catalog image and keeps unpinned ones upstream", () => { + expect(toSlimImage("pg", "supabase/postgres:17.11.0.002-orioledb")).toBe( + ORIOLEDB_FIXTURE_PIN_IMAGE, + ); + expect(toSlimImage("pg", "supabase/postgres:17.11.0.002")).toBeUndefined(); expect(toSlimImage("pg", "supabase/postgres:16.0.0.1-orioledb")).toBeUndefined(); + expect(toSlimImage("pg", "supabase/postgres:orioledb-15.1.0.55")).toBeUndefined(); + }); + + it("returns undefined for aliases and tags slimCatalogPin already excludes", () => { expect(toSlimImage("kong", dockerfileServiceImageRaw("kong"))).toBeUndefined(); expect(toSlimImage("pg", "supabase/postgres")).toBeUndefined(); }); diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index d569ba7eb8..27c9c24cb9 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -40,7 +40,7 @@ The CLI's stack catalog (`packages/stack/src/Artifacts.ts`) is the single versio every consumer of a slim-capable service: the new stack, legacy `supabase start`, and legacy slim mode. Each slim-capable service pins exactly one committed slim-services release per release line it carries (`ArtifactPin`, keyed by upstream version; only postgres carries more -than one line, its default plus an additional Postgres 15 pin). Everything else is derived from +than one line, its default plus additional Postgres 15 and OrioleDB pins). Everything else is derived from that pin — never the other way around. `apps/cli/src/shared/services/Dockerfile`, and its byte-identical Go copy @@ -141,9 +141,13 @@ upgrade first and the hotfix PR is superseded and must be closed by hand. A service with a single pin has a single line, which accepts any comparable newer upstream: a Studio year rollover or a postgrest major bump moves that line forward. Only a service with -additional pins (postgres) assigns each release tag to a line by its leading version component. A -tag on no carried line, or with a version that isn't comparable, is warned about and ignored -rather than failing the run. Comparison strips a trailing `-sha-`, so two Studio builds dated +additional pins (postgres) assigns each release tag to a line by its leading version component +plus any engine-variant suffix: postgres `17.11.0.002-orioledb` is on line `17-orioledb`, separate +from stock `17`, and compares within it without the suffix. The first release of a variant line +whose stock major the catalog carries is an **add** (branch `slim-bump/-`, title +`chore(stack): add `) that inserts the line's pin; later releases hotfix or +upgrade it like any other line. Any other tag on no carried line, or with a version that isn't +comparable, is warned about and ignored rather than failing the run. Comparison strips a trailing `-sha-`, so two Studio builds dated the same day compare equal and never produce an upgrade; the manual `--release` path pins such a build. diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index b03a8b4c2f..c899b2e1cb 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -318,7 +318,7 @@ The backend provides mechanical operations: launch, observe exit/logs, stop and A successful launch returns an owned runtime handle with readiness observation and cleanup. One shared readiness program belongs to each runtime launch and has a bounded outcome. Readiness timeout or initialization failure leaves the process running with unhealthy status and an actionable error. It does not automatically stop or restart the process. Stop remains available, and traffic/dependent launches do not treat unhealthy as ready. Initialization that requires a running process belongs to that runtime session, not the start transition. For PostgreSQL, healthy means required catalog initialization and credential reconciliation have completed—not merely that TCP accepts a connection. Stopping the running instance closes that session, settling its health probes and initialization helpers as ordinary resource cleanup. There are not two competing lifecycle operations. -The database keeps only the initialization information needed to reuse its data on a normal stop/start. Mark initialization complete only after it succeeds. If an ordinary stop interrupts application initialization, the recipe must either support a safe retry or report that initialization is incomplete; do not add a generic persisted operation journal or recovery workflow. +The database keeps only the initialization information needed to reuse its data on a normal stop/start. Data is reusable only within one release line, the PostgreSQL major plus engine variant (`17`, `17-orioledb`), so same-line minor upgrades keep their data. The readiness marker records the full artifact version, from which the line is derived. A first start records the requested line before initdb runs (a native line marker in the instance root, or the Docker storage marker), so a first start interrupted before readiness resumes only on that line. `PG_VERSION` records only the major, so data with neither a readiness marker nor a recorded line counts as the stock line and an OrioleDB request refuses it. Mark initialization complete only after it succeeds. If an ordinary stop interrupts application initialization, the recipe must either support a safe retry or report that initialization is incomplete; do not add a generic persisted operation journal or recovery workflow. Unexpected process exit is observed by the living host. Immediately disable forwarding and automatic wake for that instance and record its exit result. The executor uses the ordinary serialized stop/remove cleanup for the exact owned runtime handle, reaching stopped only after cleanup succeeds. Stop cannot be a no-op while runtime resources remain; start/restart must finish that cleanup before launching a replacement. If exit occurs during an executing transition, that transition settles and the same gate orders any remaining cleanup; no transition is pre-empted. Old-handle events cannot affect a replacement. This adds no Exited lifecycle state and no cleanup scanner: the host already owns the handle and its exit observation. diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index acdf655490..24755e2fe8 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -150,6 +150,27 @@ const definitions: Readonly> = { }, }, }, + "17.11.0.002-orioledb": { + upstreamVersion: "17.11.0.002-orioledb", + revision: 0, + image: + "ghcr.io/supabase/cli/postgres:17.11.0.002-orioledb-r0@sha256:8bfda219b7748273a46c3f097df3562a73087df8170ca4be4316f8215bf6b765", + upstreamImage: "supabase/postgres:17.11.0.002-orioledb", + natives: { + "darwin-arm64": { + archive: "66670aca62cb96a5eb555bc6d0b39e1e2581247b664c75bb3325a44eac475843", + manifest: "2a0ac16cc7cfa78a44a3876cdacd8f77622a85c55458bd80fa465daaab76b478", + }, + "linux-amd64": { + archive: "a2ea7fdd73f5f516db52ae05ca6ea52d8c94423177503f42ee8c9c2dd2474104", + manifest: "71d538e058c642b6cf7277f935074c3022b45a464079f9797ab0656d2edf2bdb", + }, + "linux-arm64": { + archive: "981c0c74ca229bb1744675527599d1180a9adef85314d2d4c0e46406c5cfa0bd", + manifest: "3faab299bedc021f5e7208d761614f3ee8a3fe08439de901a94ba2889ede404f", + }, + }, + }, }, ), rest: definition( @@ -562,10 +583,38 @@ export const resolveArtifact = Effect.fn("Artifacts.resolveArtifact")(function* }; }); -/** Resolves a PostgreSQL major alias against the pinned database artifacts. */ +const ORIOLEDB_SUFFIX = "-orioledb"; + +/** PostgreSQL major of a database artifact version or major alias. */ +export const postgresMajor = (version: string): string => version.split(".")[0] ?? version; + +/** Whether a database artifact version is an OrioleDB build. */ +export const isOrioledbVersion = (version: string): boolean => version.endsWith(ORIOLEDB_SUFFIX); + +/** + * Release line of a database artifact version: its PostgreSQL major plus engine variant, such as + * `17` or `17-orioledb`. Initialized data is reusable only within one line. + */ +export const postgresLine = (version: string): string => + isOrioledbVersion(version) + ? `${postgresMajor(version)}${ORIOLEDB_SUFFIX}` + : postgresMajor(version); + +/** Database artifact version of a `db.orioledb_version` value. */ +export const orioledbPostgresVersion = (orioledbVersion: string): string => + `${orioledbVersion}${ORIOLEDB_SUFFIX}`; + +/** `db.orioledb_version` values the catalog pins an OrioleDB artifact for. */ +export const orioledbVersions = (): ReadonlyArray => + Object.keys(definitions.database.pins) + .filter(isOrioledbVersion) + .map((version) => version.slice(0, -ORIOLEDB_SUFFIX.length)); + +/** Resolves a PostgreSQL major alias against the pinned stock database artifacts. */ export const postgresVersion = (version: string): string => - Object.keys(definitions.database.pins).find((candidate) => candidate.split(".")[0] === version) ?? - version; + Object.keys(definitions.database.pins).find( + (candidate) => !isOrioledbVersion(candidate) && postgresMajor(candidate) === version, + ) ?? version; /** Service kinds in artifact catalog order. */ export const artifactServiceKinds = (): ReadonlyArray => Record.keys(definitions); @@ -573,7 +622,7 @@ export const artifactServiceKinds = (): ReadonlyArray => Record.key /** * Every catalog pin in catalog order, including additional upstream lines. `isDefault` marks the * pin `resolveArtifact` picks when no version is requested (postgres's 17.x line today); every - * other pin (postgres's 15.x additional line) carries `isDefault: false`. + * other pin (postgres's 15.x and OrioleDB lines) carries `isDefault: false`. */ export const catalogPins = (): ReadonlyArray<{ readonly service: ServiceKind; diff --git a/packages/stack/src/Artifacts.unit.test.ts b/packages/stack/src/Artifacts.unit.test.ts index 50c55fd78c..cc32c37493 100644 --- a/packages/stack/src/Artifacts.unit.test.ts +++ b/packages/stack/src/Artifacts.unit.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "@effect/vitest"; -import { catalogPins, slimImageMirrors } from "./Artifacts.ts"; +import { catalogPins, postgresLine, slimImageMirrors } from "./Artifacts.ts"; it("carries a normalized upstreamImage for every catalog pin", () => { for (const { pin } of catalogPins()) { @@ -23,3 +23,10 @@ it("rewrites a GHCR catalog image onto ECR Public and keeps the tag and digest", it("returns no mirror for an image outside the slim catalog registry", () => { expect(slimImageMirrors("public.ecr.aws/supabase/postgres:17.6.1.173")).toEqual([]); }); + +it("keeps stock and OrioleDB data on separate lines while minor bumps share one", () => { + expect(postgresLine("17.6.1.000")).toBe(postgresLine("17.11.0.002")); + expect(postgresLine("17.11.0.002-orioledb")).toBe(postgresLine("17.12.0.001-orioledb")); + expect(postgresLine("17.11.0.002-orioledb")).not.toBe(postgresLine("17.11.0.002")); + expect(postgresLine("15.19.0.002")).not.toBe(postgresLine("17.11.0.002")); +}); diff --git a/packages/stack/src/internal/artifacts.ts b/packages/stack/src/internal/artifacts.ts index 0ee05f1fac..d76e3ffb82 100644 --- a/packages/stack/src/internal/artifacts.ts +++ b/packages/stack/src/internal/artifacts.ts @@ -4,6 +4,10 @@ export { artifactServiceKinds, catalogPins, defaultRuntime, + isOrioledbVersion, + orioledbPostgresVersion, + orioledbVersions, + postgresMajor, postgresVersion, prepareNativeArtifact, resolveArtifact, diff --git a/packages/stack/src/internal/database-reuse.ts b/packages/stack/src/internal/database-reuse.ts new file mode 100644 index 0000000000..365d01cb0d --- /dev/null +++ b/packages/stack/src/internal/database-reuse.ts @@ -0,0 +1,33 @@ +import { isOrioledbVersion, postgresLine, postgresMajor } from "../Artifacts.ts"; + +/** Recovery for local database data a stack cannot reuse, worded like the CLI's saved-stack advice. */ +export const recreateStackAdvice = (stackId: string): string => + `run \`supabase stack destroy --stack-id ${stackId}\` to recreate the stack — this permanently deletes its local database data`; + +/** + * Why existing PostgreSQL data cannot serve the requested artifact version, or `undefined` when it + * can. `PG_VERSION` proves only the major, so an unknown `line` counts as the stock line. + */ +export const unusableDatabaseData = ( + data: { + readonly major: string; + readonly line: string | undefined; + readonly initialized: boolean; + }, + version: string, +): string | undefined => { + const subject = data.initialized + ? "Initialized PostgreSQL data" + : "PostgreSQL data from an unfinished first start"; + const major = postgresMajor(version); + if (data.major !== major) + return `${subject} is major ${data.major}, but major ${major} was requested`; + const line = postgresLine(version); + if (data.line === undefined) + return isOrioledbVersion(version) + ? "Unmarked PostgreSQL data cannot be verified as OrioleDB data" + : undefined; + return data.line === line + ? undefined + : `${subject} belongs to release line ${data.line}, but ${line} was requested`; +}; diff --git a/packages/stack/src/internal/database-reuse.unit.test.ts b/packages/stack/src/internal/database-reuse.unit.test.ts new file mode 100644 index 0000000000..834e241ecd --- /dev/null +++ b/packages/stack/src/internal/database-reuse.unit.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "@effect/vitest"; +import { unusableDatabaseData } from "./database-reuse.ts"; + +const stock = "17.11.0.002"; +const orioledb = "17.11.0.002-orioledb"; + +describe("unusableDatabaseData", () => { + it.each([ + { + name: "initialized data from another major", + data: { major: "15", line: "15", initialized: true }, + version: stock, + reason: "Initialized PostgreSQL data is major 15, but major 17 was requested", + }, + { + name: "unfinished data from another major", + data: { major: "15", line: undefined, initialized: false }, + version: stock, + reason: + "PostgreSQL data from an unfinished first start is major 15, but major 17 was requested", + }, + { + name: "unrecorded data for OrioleDB", + data: { major: "17", line: undefined, initialized: false }, + version: orioledb, + reason: "Unmarked PostgreSQL data cannot be verified as OrioleDB data", + }, + { + name: "stock-recorded data for OrioleDB", + data: { major: "17", line: "17", initialized: false }, + version: orioledb, + reason: + "PostgreSQL data from an unfinished first start belongs to release line 17, but 17-orioledb was requested", + }, + { + name: "OrioleDB-recorded data for stock", + data: { major: "17", line: "17-orioledb", initialized: false }, + version: stock, + reason: + "PostgreSQL data from an unfinished first start belongs to release line 17-orioledb, but 17 was requested", + }, + ])("refuses $name", ({ data, version, reason }) => { + expect(unusableDatabaseData(data, version)).toBe(reason); + }); + + it.each([ + { name: "unrecorded data for stock", line: undefined, version: stock }, + { name: "stock-recorded data for stock", line: "17", version: stock }, + { name: "OrioleDB-recorded data for OrioleDB", line: "17-orioledb", version: orioledb }, + ])("reuses $name", ({ line, version }) => { + expect( + unusableDatabaseData({ major: "17", line, initialized: false }, version), + ).toBeUndefined(); + }); +}); diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts index 08afa7397d..6d72d6f2ad 100644 --- a/packages/stack/src/services/Database.integration.test.ts +++ b/packages/stack/src/services/Database.integration.test.ts @@ -14,6 +14,7 @@ import { Stream, } from "effect"; import { tmpdir } from "node:os"; +import { postgresVersion } from "../Artifacts.ts"; import { DEFAULT_POSTGRES_ROOT_KEY } from "../Defaults.ts"; import { makeService } from "../Service.ts"; import { makeDatabase, type BackendEndpoint, type DatabaseConfig } from "./Database.ts"; @@ -156,6 +157,100 @@ describe("database component", { timeout: 180_000 }, () => { ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); + it.live("refuses initialized data from another engine line before preparing an artifact", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-line-" }); + const recipe = yield* makeDatabase({ + stackId: "database-line-test", + instanceId: "database", + root, + cacheRoot: artifactCacheRoot, + runtime: "native", + }); + const prepare = recipe.definition.prepare; + if (prepare === undefined) return yield* Effect.die("database recipe has no prepare"); + const instanceRoot = path.join(root, "database"); + const orioledb: DatabaseConfig = { ...config, version: "17.11.0.002-orioledb" }; + const recreate = + "run `supabase stack destroy --stack-id database-line-test` to recreate the stack — this permanently deletes its local database data"; + yield* fs.makeDirectory(path.join(instanceRoot, "data"), { recursive: true }); + yield* fs.writeFileString(path.join(instanceRoot, "data", "PG_VERSION"), "15\n"); + + expect((yield* Effect.flip(prepare(config))).message).toContain( + `PostgreSQL data from an unfinished first start is major 15, but major 17 was requested; ${recreate}`, + ); + + yield* fs.writeFileString(path.join(instanceRoot, "data", "PG_VERSION"), "17\n"); + expect((yield* Effect.flip(prepare(orioledb))).message).toContain( + `Unmarked PostgreSQL data cannot be verified as OrioleDB data; ${recreate}`, + ); + + yield* fs.writeFileString( + path.join(instanceRoot, ".supabase-database-line.json"), + '{"line":"17-orioledb"}', + ); + expect((yield* Effect.flip(prepare(config))).message).toContain( + `PostgreSQL data from an unfinished first start belongs to release line 17-orioledb, but 17 was requested; ${recreate}`, + ); + + const marker = (version: string) => + fs.writeFileString( + path.join(instanceRoot, ".supabase-database-ready.json"), + JSON.stringify({ version, runtime: "native", profile: "supabase" }), + ); + yield* marker("17.11.0.002"); + expect((yield* Effect.flip(prepare(orioledb))).message).toContain( + `Initialized database data is 17.11.0.002 on the native runtime, but 17.11.0.002-orioledb on the native runtime was requested; ${recreate}`, + ); + yield* marker("17.11.0.002-orioledb"); + expect((yield* Effect.flip(prepare(config))).message).toContain( + "Initialized database data is 17.11.0.002-orioledb on the native runtime", + ); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + + it.live("resumes an interrupted stock native first start and refuses it for OrioleDB", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-resume-" }); + const recipe = yield* makeDatabase({ + stackId: "database-resume-test", + instanceId: "database", + root, + cacheRoot: artifactCacheRoot, + runtime: "native", + }); + const prepare = recipe.definition.prepare; + if (prepare === undefined) return yield* Effect.die("database recipe has no prepare"); + const service = yield* makeService(recipe.definition, { + id: "database", + config: { ...config, healthTimeoutMs: 120_000 }, + }); + yield* service.start; + yield* service.ready; + yield* service.stop; + // Initialized data without its readiness marker is what an interrupted first start leaves. + yield* fs.remove(path.join(root, "database", ".supabase-database-ready.json")); + + expect( + (yield* Effect.flip(prepare({ ...config, version: "17.11.0.002-orioledb" }))).message, + ).toContain("belongs to release line 17, but 17-orioledb was requested"); + yield* service.start; + yield* service.ready; + expect(yield* fs.exists(path.join(root, "database", ".supabase-database-ready.json"))).toBe( + true, + ); + yield* service.destroy; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + it.live("requires passwords from non-superusers on the native socket", () => Effect.scoped( Effect.gen(function* () { @@ -289,12 +384,16 @@ describe("database component", { timeout: 180_000 }, () => { yield* service.restart({ ...config, version: "unsupported" }).pipe(Effect.flip); expect((yield* service.get).lifecycle).toBe("running"); const mismatch = yield* service.restart({ ...config, version: "15" }).pipe(Effect.flip); - expect(mismatch.message).toContain("does not match"); + expect(mismatch.message).toContain( + `Initialized database data is ${postgresVersion("17")} on the native runtime, but ${postgresVersion("15")} on the native runtime was requested`, + ); expect((yield* service.get).lifecycle).toBe("running"); yield* service.stop; yield* fs.remove(path.join(root, "first", ".supabase-database-ready.json")); const incomplete = yield* service.restart({ ...config, version: "15" }).pipe(Effect.flip); - expect(incomplete.message).toContain("major does not match"); + expect(incomplete.message).toContain( + "PostgreSQL data from an unfinished first start is major 17, but major 15 was requested", + ); const reopened = yield* makeDatabase({ stackId: "stack-integration", diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 3462e19b05..2beb81af51 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -24,6 +24,7 @@ import { HttpClient } from "effect/unstable/http"; import { slimImageMirrors, prepareNativeArtifact, + postgresLine, postgresVersion, resolveArtifact, type PreparedNativeArtifact, @@ -82,6 +83,7 @@ import { type DockerDatabaseStorage, type DockerDatabaseStorageError, } from "../storage/DockerDatabaseStorage.ts"; +import { recreateStackAdvice, unusableDatabaseData } from "../internal/database-reuse.ts"; export const DatabaseConfig = Schema.Struct({ version: Schema.String, @@ -103,6 +105,10 @@ const DatabaseReadyMarker = Schema.Struct({ profile: Schema.Literal("supabase"), }); +/** Release line a native first start initializes, written before initdb so an interrupted start can resume. */ +const DatabaseLineMarker = Schema.Struct({ line: Schema.String }); +const lineMarkerFile = ".supabase-database-line.json"; + // Health reconciles role passwords as supabase_admin, including after a configured password change. const NATIVE_HBA_RULES = "local all supabase_admin trust\nlocal all all scram-sha-256\n"; @@ -652,8 +658,28 @@ export const makeDatabase = ( ); }); + /** Records the release line before initdb runs; initialized data keeps the line it has. */ + const recordLine = Effect.fn("Database.recordLine")( + function* (dataPath: string, version: string) { + if (yield* fs.exists(path.join(dataPath, "PG_VERSION"))) return; + const encoded = yield* Schema.encodeEffect(Schema.fromJsonString(DatabaseLineMarker))({ + line: postgresLine(version), + }); + const stage = yield* fs.makeTempDirectoryScoped({ + directory: instanceRoot, + prefix: ".line-", + }); + yield* fs.writeFileString(path.join(stage, "marker"), encoded, { mode: 0o600 }); + yield* fs.rename(path.join(stage, "marker"), path.join(instanceRoot, lineMarkerFile)); + }, + Effect.scoped, + Effect.mapError((cause) => errorFor("launch", cause)), + ); + const prepare = Effect.fn("Database.prepare")( function* (input: DatabaseConfig) { + const requested = postgresVersion(input.version); + const recreate = recreateStackAdvice(String(options.stackId)); const markerPath = path.join(instanceRoot, ".supabase-database-ready.json"); const hasMarker = yield* fs.exists(markerPath); if (hasMarker) { @@ -661,22 +687,33 @@ export const makeDatabase = ( .readFileString(markerPath) .pipe(Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(DatabaseReadyMarker)))); if ( - marker.version.split(".")[0] !== postgresVersion(input.version).split(".")[0] || + postgresLine(marker.version) !== postgresLine(requested) || marker.runtime !== options.runtime ) return yield* errorFor( "prepare", - "Initialized database artifact/runtime does not match the requested configuration", + `Initialized database data is ${marker.version} on the ${marker.runtime} runtime, but ${requested} on the ${options.runtime} runtime was requested; ${recreate}`, ); } const versionPath = path.join(instanceRoot, "data", "PG_VERSION"); if (!hasMarker && options.runtime === "native" && (yield* fs.exists(versionPath))) { - const initialized = (yield* fs.readFileString(versionPath)).trim(); - if (initialized !== postgresVersion(input.version).split(".")[0]) - return yield* errorFor( - "prepare", - "Initialized PostgreSQL major does not match the requested configuration", - ); + const linePath = path.join(instanceRoot, lineMarkerFile); + const recorded = (yield* fs.exists(linePath)) + ? (yield* fs + .readFileString(linePath) + .pipe( + Effect.flatMap(Schema.decodeEffect(Schema.fromJsonString(DatabaseLineMarker))), + )).line + : undefined; + const reason = unusableDatabaseData( + { + major: (yield* fs.readFileString(versionPath)).trim(), + line: recorded, + initialized: false, + }, + requested, + ); + if (reason !== undefined) return yield* errorFor("prepare", `${reason}; ${recreate}`); } yield* prepareArtifact(input); }, @@ -815,6 +852,7 @@ export const makeDatabase = ( yield* fs .makeDirectory(dataPath, { recursive: true, mode: 0o700 }) .pipe(Effect.mapError((cause) => errorFor("launch", cause))); + yield* recordLine(dataPath, config.version); const rootKeyPath = path.join(nativeRoot, "pgsodium_root.key"); yield* fs .writeFileString(rootKeyPath, Redacted.value(config.rootKey), { mode: 0o600 }) diff --git a/packages/stack/src/services/DatabaseSnapshot.ts b/packages/stack/src/services/DatabaseSnapshot.ts index eb335fcaa1..7f7c0022b1 100644 --- a/packages/stack/src/services/DatabaseSnapshot.ts +++ b/packages/stack/src/services/DatabaseSnapshot.ts @@ -15,7 +15,7 @@ import { } from "effect"; import type { PlatformError } from "effect/PlatformError"; import { ChildProcessSpawner } from "effect/unstable/process"; -import { postgresVersion } from "../Artifacts.ts"; +import { postgresMajor, postgresVersion } from "../Artifacts.ts"; import { failureMessage } from "../internal/failure-message.ts"; import { copyDirectory, type DirectoryCopyError } from "../storage/DirectoryCopy.ts"; import type { DatabaseRuntime } from "./Database.ts"; @@ -161,7 +161,9 @@ export const makeSnapshotStore = Effect.fn("DatabaseSnapshot.makeStore")(functio const { backends, runtime, version } = options; const { Adopt, Clear, Copy, Ensure, Expect, ExpectEmpty, ExpectText } = SnapshotStep; const { Prune, Recover, Remove, Rename, Touch, Write } = SnapshotStep; - const major = version.split(".")[0] ?? version; + // PG_VERSION holds only the major; the descriptor and ready marker carry the full version, so + // snapshots never cross release lines. + const major = postgresMajor(version); const markerPath = path.join(options.instanceRoot, ".supabase-database-ready.json"); const mapError = (operation: string, effect: Effect.Effect) => effect.pipe(Effect.mapError((cause) => errorFor(operation, cause))); diff --git a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts index 16619a0b77..f40b217964 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.integration.test.ts @@ -38,6 +38,7 @@ const Marker = Schema.Struct({ cacheNamespace: Schema.String, daemonId: Schema.optionalKey(Schema.String), initialized: Schema.Boolean, + line: Schema.optionalKey(Schema.String), }); class DockerTestError extends Data.TaggedError("DockerTestError")<{ @@ -82,7 +83,8 @@ const fakeHelperEngine = () => { ? Effect.succeed(handle(0, "abcdef0123456789")) : Effect.succeed(handle(1, "", `Unable to find image '${image ?? ""}' locally`)); } - if (args[0] === "exec" || args[0] === "rm") return Effect.succeed(handle(0, "done")); + if (args[0] === "exec") return Effect.succeed(handle(0)); + if (args[0] === "rm") return Effect.succeed(handle(0, "done")); return Effect.succeed(handle(1, "", `unexpected engine command: ${args[0] ?? ""}`)); }); return { commands, layer: Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner) }; @@ -1025,6 +1027,197 @@ describe("Docker database storage", { timeout: 120_000 }, () => { ).pipe(Effect.provide(NodeServices.layer)), ); + it.live("resumes unfinished volume data only on the release line its first start recorded", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const helperImage = yield* postgresImage("17"); + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-line-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "line"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + stackId: `storage-line-${yield* crypto.randomUUIDv4}`, + instanceId: "line", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container: yield* makeContainerRuntime({ engine: "docker", root }), + spawner, + }); + let volume: string | undefined; + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + yield* storage.destroyData("17").pipe(Effect.ignore); + if (volume !== undefined) yield* docker(["volume", "rm", volume]).pipe(Effect.ignore); + }).pipe(Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner)), + ); + yield* storage.prepare("17"); + const marker = yield* Schema.decodeEffect(Schema.fromJsonString(Marker))( + yield* fs.readFileString(path.join(instanceRoot, ".supabase-database-storage.json")), + ); + volume = marker.volume; + if (marker.backend !== "docker" || volume === undefined) + return yield* new DockerTestError({ message: "Docker test selected host fallback" }); + // Stands in for initdb output that an interrupted first start leaves without readiness. + const writePgVersion = docker([ + "run", + "--rm", + "--mount", + `type=volume,src=${volume},dst=/store`, + helperImage, + "/bin/sh", + "-c", + `printf 17 > ${quote(`/store/${marker.namespace}/data/PG_VERSION`)}`, + ]); + yield* writePgVersion; + + const oriole = "17.11.0.002-orioledb"; + expect((yield* storage.prepare(oriole).pipe(Effect.flip)).message).toContain( + "PostgreSQL data from an unfinished first start belongs to release line 17, but 17-orioledb was requested", + ); + yield* storage.prepare("17"); + + yield* storage.removeData("17"); + yield* storage.prepare(oriole); + yield* writePgVersion; + expect((yield* storage.prepare("17").pipe(Effect.flip)).message).toContain( + "PostgreSQL data from an unfinished first start belongs to release line 17-orioledb, but 17 was requested", + ); + yield* storage.prepare(oriole); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live( + "refuses unfinished host data without a recorded line for OrioleDB and resumes it as stock", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-host-line-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "line"); + yield* fs.makeDirectory(path.join(instanceRoot, "data"), { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + // A recorded host marker keeps the bind-mount backend even on volume-capable engines. + yield* fs.writeFileString( + path.join(instanceRoot, ".supabase-database-storage.json"), + yield* Schema.encodeEffect(Schema.fromJsonString(Marker))({ + backend: "host", + namespace: "instance-storage-host-line-line", + cacheNamespace: `cache-${"0".repeat(32)}`, + initialized: false, + }), + { mode: 0o600 }, + ); + yield* fs.writeFileString(path.join(instanceRoot, "data", "PG_VERSION"), "17\n"); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + stackId: "storage-host-line", + instanceId: "line", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container: yield* makeContainerRuntime({ engine: "docker", root }), + spawner, + }); + yield* Effect.addFinalizer(() => + storage + .destroyData("17") + .pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.ignore, + ), + ); + + const failure = yield* storage.prepare("17.11.0.002-orioledb").pipe(Effect.flip); + expect(failure.message).toContain( + "Unmarked PostgreSQL data cannot be verified as OrioleDB data; run `supabase stack destroy --stack-id storage-host-line` to recreate the stack — this permanently deletes its local database data", + ); + yield* storage.prepare("17"); + expect(yield* fs.readFileString(path.join(instanceRoot, "data", "PG_VERSION"))).toBe( + "17\n", + ); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + + it.live("resumes unfinished host data only on the release line its first start recorded", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "docker-storage-host-resume-" }); + const storageRoot = path.join(root, "state", "stack", "data"); + const cacheRoot = path.join(root, "cache"); + const instanceRoot = path.join(storageRoot, "resume"); + yield* fs.makeDirectory(instanceRoot, { recursive: true }); + yield* fs.makeDirectory(cacheRoot, { recursive: true }); + // A marker written before release lines were recorded gains one on its first start. + yield* fs.writeFileString( + path.join(instanceRoot, ".supabase-database-storage.json"), + yield* Schema.encodeEffect(Schema.fromJsonString(Marker))({ + backend: "host", + namespace: "instance-storage-host-resume-resume", + cacheNamespace: `cache-${"0".repeat(32)}`, + initialized: false, + }), + { mode: 0o600 }, + ); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const storage = yield* makeDockerDatabaseStorage({ + runtime: "docker", + stackId: "storage-host-resume", + instanceId: "resume", + instanceRoot, + root: storageRoot, + cacheRoot, + fs, + path, + crypto, + container: yield* makeContainerRuntime({ engine: "docker", root }), + spawner, + }); + yield* Effect.addFinalizer(() => + storage + .destroyData("17") + .pipe( + Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.ignore, + ), + ); + + const oriole = "17.11.0.002-orioledb"; + yield* storage.prepare(oriole); + // Stands in for initdb output that an interrupted first start leaves without readiness. + yield* fs.writeFileString(path.join(instanceRoot, "data", "PG_VERSION"), "17\n"); + + expect((yield* storage.prepare("17").pipe(Effect.flip)).message).toContain( + "PostgreSQL data from an unfinished first start belongs to release line 17-orioledb, but 17 was requested", + ); + yield* storage.prepare(oriole); + }), + ).pipe(Effect.provide(NodeServices.layer)), + ); + it.live("handles root-owned host data through helper operations", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/storage/DockerDatabaseStorage.ts b/packages/stack/src/storage/DockerDatabaseStorage.ts index f35f5d9a4f..732f4f55fc 100644 --- a/packages/stack/src/storage/DockerDatabaseStorage.ts +++ b/packages/stack/src/storage/DockerDatabaseStorage.ts @@ -13,7 +13,8 @@ import { } from "effect"; import { ChildProcess } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; -import { postgresVersion, resolveArtifact } from "../Artifacts.ts"; +import { postgresLine, postgresMajor, postgresVersion, resolveArtifact } from "../Artifacts.ts"; +import { recreateStackAdvice, unusableDatabaseData } from "../internal/database-reuse.ts"; import { failureMessage } from "../internal/failure-message.ts"; import { testRunLabelArgs as readTestRunLabelArgs } from "../internal/test-run-label.ts"; import type { ContainerRuntime } from "../runtime/Container.ts"; @@ -35,6 +36,8 @@ const Marker = Schema.Struct({ cacheNamespace: Schema.String, daemonId: Schema.optionalKey(Schema.String), initialized: Schema.Boolean, + /** Release line recorded before initdb; markers written before this field have none. */ + line: Schema.optionalKey(Schema.String), }); type Marker = Schema.Schema.Type; const DaemonIdentity = Schema.Struct({ @@ -834,6 +837,28 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") options.fs.writeFileString(markerPath, encoded, { mode: 0o600 }), ), ); + const checkInitialized = Effect.fnUntraced(function* ( + pgVersion: string, + version: string, + marker: Marker, + ) { + // The database checks a present readiness marker's line before preparing storage. + const verified = yield* options.fs.exists(readyMarkerPath); + const reason = unusableDatabaseData( + { + major: pgVersion.trim(), + line: verified ? postgresLine(version) : marker.line, + initialized: marker.initialized, + }, + version, + ); + if (reason !== undefined) + return yield* errorFor("prepare", `${reason}; ${recreateStackAdvice(options.stackId)}`); + }); + const recordLine = (marker: Marker, version: string) => + marker.line === postgresLine(version) + ? Effect.void + : writeMarker({ ...marker, line: postgresLine(version) }); const setup = Effect.fn("DockerDatabaseStorage.prepare")((version: string) => Effect.gen(function* () { yield* selected; @@ -851,13 +876,20 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") ], version, ); - if (major.trim() !== majorVersion(version)) - return yield* errorFor( - "prepare", - "Initialized PostgreSQL major does not match the requested configuration", - ); + yield* checkInitialized(major, version, marker); + } else if (yield* options.fs.exists(data)) { + // An interrupted first start can leave initdb output behind without readiness. + const major = yield* runHelper( + "set -eu; if [ -f /instance/data/PG_VERSION ]; then cat /instance/data/PG_VERSION; fi", + [{ source: options.instanceRoot, target: "/instance", readOnly: false }], + version, + true, + ); + if (major.trim() !== "") yield* checkInitialized(major, version, marker); + else yield* recordLine(marker, version); } else { yield* options.fs.makeDirectory(data, { recursive: true, mode: 0o700 }); + yield* recordLine(marker, version); } return; } @@ -869,22 +901,19 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") [{ source: marker.volume ?? "", target: "/store", readOnly: false, type: "volume" }], version, ); - if (major.trim() !== majorVersion(version)) - return yield* errorFor( - "prepare", - "Initialized PostgreSQL major does not match the requested configuration", - ); + yield* checkInitialized(major, version, marker); } else { - yield* runHelper( - `set -eu; mkdir -p ${shellQuote(`${store}/data`)} ${shellQuote(`${cache}/entries`)} ${shellQuote(`${cache}/stages`)}; chown -R 100:101 ${shellQuote(`${store}/data`)}`, + const major = yield* runHelper( + `set -eu; if [ -f ${shellQuote(`${store}/data/PG_VERSION`)} ]; then cat ${shellQuote(`${store}/data/PG_VERSION`)}; fi; mkdir -p ${shellQuote(`${store}/data`)} ${shellQuote(`${cache}/entries`)} ${shellQuote(`${cache}/stages`)}; chown -R 100:101 ${shellQuote(`${store}/data`)}`, [{ source: marker.volume ?? "", target: "/store", readOnly: false, type: "volume" }], version, + true, ); + if (major.trim() !== "") yield* checkInitialized(major, version, marker); + else yield* recordLine(marker, version); } }).pipe(Effect.mapError((cause) => errorFor("prepare", cause))), ); - const majorVersion = (version: string) => version.split(".")[0] ?? version; - const mount = (_version: string) => selected.pipe( Effect.flatMap(() => getMarker), @@ -916,7 +945,7 @@ export const makeDockerDatabaseStorage = Effect.fn("DockerDatabaseStorage.make") if (marker.backend === "docker") { const versionFile = `/store/${marker.namespace}/data/PG_VERSION`; yield* runHelper( - `set -eu; if [ ! -f ${shellQuote(versionFile)} ]; then echo 'Database readiness requires PG_VERSION' >&2; exit 1; fi; actual=$(cat ${shellQuote(versionFile)}); if [ "$actual" != ${shellQuote(majorVersion(version))} ]; then echo 'Database PostgreSQL major does not match requested version' >&2; exit 1; fi`, + `set -eu; if [ ! -f ${shellQuote(versionFile)} ]; then echo 'Database readiness requires PG_VERSION' >&2; exit 1; fi; actual=$(cat ${shellQuote(versionFile)}); if [ "$actual" != ${shellQuote(postgresMajor(version))} ]; then echo 'Database PostgreSQL major does not match requested version' >&2; exit 1; fi`, snapshotPaths(marker).mounts, version, );