diff --git a/apps/cli/src/commands/experimental/compute/new/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/compute/new/SIDE_EFFECTS.md index 3a1e04bdf9..872a540304 100644 --- a/apps/cli/src/commands/experimental/compute/new/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/compute/new/SIDE_EFFECTS.md @@ -24,12 +24,29 @@ and the command handler does not run. See the [Compute command guide](../../../. ## Files Written -| Path | Format | When | -| ----------------------------------------------- | ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `/supabase/config.toml` | TOML | on success — appends `[compute.]` with `runtime`, `size` and `exposure` always, `instances` only when it differs from the default of 1, and `source` only when `--source` was passed, preserving surrounding formatting | -| `/supabase/compute//*` | varies | on success, unless `--source` names another directory | -| `//*` | varies | on success, when `--source` is given | -| `/telemetry.json` | JSON | whenever the handler runs — flushed on success and on failure | +| Path | Format | When | +| ----------------------------------------------- | ------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/supabase/config.toml` | TOML | on success — appends `[compute.]` with `runtime`, `size` and `exposure` always, `exclude` whenever the chosen runtime declares default patterns, `instances` only when it differs from the default of 1, and `source` only when `--source` was passed, preserving surrounding formatting | +| `/supabase/compute//*` | varies | on success, unless `--source` names another directory | +| `//*` | varies | on success, when `--source` is given | +| `/telemetry.json` | JSON | whenever the handler runs — flushed on success and on failure | + +## Default `exclude` patterns + +Each runtime declares the `[compute.] exclude` patterns a scaffold starts with, and they +are written into `config.toml` as a single-line TOML array rather than applied silently at push +time — the list is the runtime's opinion about its own build, and a file the user can read and +edit is the only place that opinion can be argued with. `push` has no built-in defaults of its +own, so editing or deleting the line is all it takes to change what ships. + +Every runtime excludes environment files and version-control metadata (`.env`, `.env.*`, +`.git`, matching a worktree's `.git` file as well as a repository's directory). Beyond that the two catalog runtimes carry more than `dockerfile` does, because the +CLI knows what tooling writes into their directories: `node` also drops `node_modules/` and +`*.log`, `deno` drops `*.log`, and a `dockerfile` context is left alone because the user's own +`Dockerfile` already decides what it copies. The key is +omitted entirely for a runtime that declares no patterns. The chosen list is reported as the +`Excluded` row beside the compute's other dials, and carried on the machine-output payload as +`exclude`. Compute resources are recorded in `config.toml` only. The project config loader prefers `supabase/config.json` when one exists, but the entry writer is a TOML text diff --git a/apps/cli/src/commands/experimental/compute/new/new.handler.ts b/apps/cli/src/commands/experimental/compute/new/new.handler.ts index e62b9ffb31..b19a971af2 100644 --- a/apps/cli/src/commands/experimental/compute/new/new.handler.ts +++ b/apps/cli/src/commands/experimental/compute/new/new.handler.ts @@ -32,6 +32,7 @@ import { COMPUTE_EXPOSURE_DESCRIPTIONS, COMPUTE_EXPOSURES, COMPUTE_RUNTIME_DESCRIPTIONS, + COMPUTE_RUNTIME_EXCLUSIONS, COMPUTE_RUNTIMES, COMPUTE_SIZES, type ComputeExposure, @@ -273,6 +274,10 @@ export const computeNew = Effect.fn("compute.new")(function* (flags: ComputeNewF const size = yield* resolveSize({ explicit: flags.size, canPrompt }); const exposure = yield* resolveExposure({ explicit: flags.exposure, canPrompt }); const instances = recordedInstances(flags.instances); + // The chosen runtime's own defaults, written down rather than applied invisibly at push + // time: the list is the runtime's opinion about its build, and a `config.toml` the user + // can read and edit is the only place that opinion can be argued with. + const exclude = COMPUTE_RUNTIME_EXCLUSIONS[runtime]; // Validated before anything is written: this is the directory the starter files // land in, so a value naming the project root, `supabase/`, or anywhere outside @@ -331,6 +336,7 @@ export const computeNew = Effect.fn("compute.new")(function* (flags: ComputeNewF exposure, ...(instances === undefined ? {} : { instances }), ...(source === undefined ? {} : { source }), + ...(exclude.length === 0 ? {} : { exclude }), }, }); @@ -363,6 +369,7 @@ export const computeNew = Effect.fn("compute.new")(function* (flags: ComputeNewF // than "one". instances: instances ?? DEFAULT_COMPUTE_INSTANCES, source: sourceDisplay, + exclude, config_path: project.configPath, }; @@ -391,6 +398,9 @@ export const computeNew = Effect.fn("compute.new")(function* (flags: ComputeNewF // `declared`, the way `compute status` labels the same number: nothing // is running yet, so a bare count would read as a live tally. ["Instances", `${instances ?? DEFAULT_COMPUTE_INSTANCES} declared`], + // Shown because the scaffold decided it: a compute that silently leaves files out of + // its deploy should say so where the rest of its dials are reported. + ["Excluded", exclude.join(", ")], ]), ); // On the success trailer rather than inline, the way `bootstrap` emits its diff --git a/apps/cli/src/commands/experimental/compute/new/new.integration.test.ts b/apps/cli/src/commands/experimental/compute/new/new.integration.test.ts index ae1ba17456..bcc32642be 100644 --- a/apps/cli/src/commands/experimental/compute/new/new.integration.test.ts +++ b/apps/cli/src/commands/experimental/compute/new/new.integration.test.ts @@ -1,6 +1,7 @@ import { BunServices } from "@effect/platform-bun"; import { describe, expect, it } from "@effect/vitest"; import { Effect, Option, FileSystem, Path, Predicate, Schema } from "effect"; +import * as SmolToml from "smol-toml"; import { makeComputeProject, setupCompute } from "../../../../../tests/helpers/compute.ts"; import { ComputeAlreadyConfiguredError, @@ -13,6 +14,10 @@ import { ComputeDirectoryExistsError, ComputeJsonConfigUnsupportedError, } from "../../../../shared/compute/compute.errors.ts"; +import { + COMPUTE_RUNTIME_EXCLUSIONS, + type ComputeRuntime, +} from "../../../../shared/compute/compute-runtimes.ts"; import { computeNew } from "./new.handler.ts"; import { ComputeNewWorkdirError } from "./new.errors.ts"; import type { ComputeNewFlags } from "./new.command.ts"; @@ -24,6 +29,18 @@ project_id = "demo" verify_jwt = false `; +/** + * The `exclude = [...]` line `new` writes for a runtime, derived from the runtime's own list + * rather than restated: which patterns a runtime chooses is asserted where the patterns are + * matched, and the claim here is only that the list reaches `config.toml` as a TOML array. + */ +function excludeLine(runtime: ComputeRuntime): string { + const patterns = COMPUTE_RUNTIME_EXCLUSIONS[runtime]; + return patterns.length === 0 + ? "" + : `exclude = [${patterns.map((pattern) => `"${pattern}"`).join(", ")}]\n`; +} + function flags(overrides: Partial = {}): ComputeNewFlags { return { name: Option.some("api"), @@ -63,7 +80,9 @@ describe("compute new", () => { const computeDir = path.join(repo.dir, "supabase", "compute", "api"); expect(yield* fs.exists(path.join(computeDir, "index.mjs"))).toBe(true); expect(yield* repo.config).toBe( - `${CONFIG_WITH_COMMENTS}\n[compute.api]\nruntime = "node"\nsize = "2gb"\nexposure = "public"\n`, + `${CONFIG_WITH_COMMENTS}\n[compute.api]\nruntime = "node"\nsize = "2gb"\nexposure = "public"\n${excludeLine( + "node", + )}`, ); // Declarative line first, then the detail rows, then the next step — @@ -308,6 +327,58 @@ describe("compute new", () => { }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); + // Written into config.toml rather than applied invisibly at push time, so the list is + // visible and editable and `push` needs no built-in defaults of its own. + describe("the chosen runtime's default exclude patterns", () => { + it.live.each(["node", "deno", "dockerfile"] as const)( + "records the %s runtime's own list", + (runtime) => + Effect.gen(function* () { + const repo = yield* project(); + const { layer } = setupCompute({ workdir: repo.dir }); + + return yield* Effect.gen(function* () { + yield* computeNew(flags({ runtime: Option.some(runtime) })); + + expect(yield* repo.config).toContain(excludeLine(runtime).trimEnd()); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + // The written entry has to be loadable, or the scaffold leaves behind a project whose + // config nothing can read — the patterns are quoted strings in a TOML array, which is + // exactly the shape a hand-rolled renderer gets wrong. + it.live("writes them as a list the config loader reads back", () => + Effect.gen(function* () { + const repo = yield* project(); + const { layer } = setupCompute({ workdir: repo.dir }); + + return yield* Effect.gen(function* () { + yield* computeNew(flags({ runtime: Option.some("node") })); + + const parsed = SmolToml.parse(yield* repo.config) as { + compute?: { api?: { exclude?: unknown } }; + }; + expect(parsed.compute?.api?.exclude).toEqual([...COMPUTE_RUNTIME_EXCLUSIONS.node]); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + it.live("reports them alongside the compute's other dials", () => + Effect.gen(function* () { + const repo = yield* project(); + const { layer, out } = setupCompute({ workdir: repo.dir }); + + return yield* Effect.gen(function* () { + yield* computeNew(flags({ runtime: Option.some("node") })); + + expect(out.stdoutText).toContain("Excluded"); + expect(out.stdoutText).toContain(".env"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + }); + // The runtime and size prompts do have defaults to fall back on, so a piped // stdin must leave them unasked rather than consuming the pipe. it.live("takes the defaults without prompting when stdin is piped", () => @@ -517,7 +588,7 @@ describe("compute new", () => { const computeDir = path.join(created.dir, "supabase", "compute", "api"); expect(yield* fs.exists(path.join(computeDir, "index.mjs"))).toBe(true); expect(yield* fs.readFileString(path.join(created.dir, "supabase", "config.toml"))).toBe( - `[compute.api]\nruntime = "node"\nsize = "2gb"\nexposure = "public"\n`, + `[compute.api]\nruntime = "node"\nsize = "2gb"\nexposure = "public"\n${excludeLine("node")}`, ); // An EXPLICIT --workdir has no cwd-relative reading, so the success // message names the absolute path rather than a project-root-relative one. @@ -783,7 +854,7 @@ describe("compute new", () => { // The workdir got both the entry and the scaffold it points at. expect(yield* fs.readFileString(path.join(workdir, "supabase", "config.toml"))).toBe( - '[compute.api]\nruntime = "node"\nsize = "2gb"\nexposure = "public"\n', + `[compute.api]\nruntime = "node"\nsize = "2gb"\nexposure = "public"\n${excludeLine("node")}`, ); expect( yield* fs.exists(path.join(workdir, "supabase", "compute", "api", "index.mjs")), diff --git a/apps/cli/src/commands/experimental/compute/push/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/compute/push/SIDE_EFFECTS.md index 1201d594f9..67f60895be 100644 --- a/apps/cli/src/commands/experimental/compute/push/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/compute/push/SIDE_EFFECTS.md @@ -20,7 +20,7 @@ and the command handler does not run. See the [Compute command guide](../../../. | ---------------------------------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `/supabase/config.json` | JSON | always when present — preferred over `config.toml`; each compute's runtime, size, exposure, instances, source. An explicit `--workdir`/`SUPABASE_WORKDIR` is read exactly as given, with no ancestor search; with a DEFAULTED workdir the loader may resolve an ancestor project's `config.json` from a subdirectory (CLI-2285), and the resolved compute `source` below resolves against that SAME ancestor root | | `/supabase/config.toml` | TOML | always when no `config.json` exists — the same compute fields, with the same explicit-vs-default workdir rule | -| `/**` | any | always — packaged into the build context | +| `/**` | any | always — packaged into the build context, minus whatever `[compute.] exclude` matches. An excluded directory is not descended into, so nothing under it is read at all | | `/profile` | plain text | when neither `--profile` nor `SUPABASE_PROFILE` is set — names the profile, defaulting to `supabase` | | `` (YAML) | YAML | when `SUPABASE_PROFILE` is a filesystem path rather than a built-in name; a read failure aborts the command | @@ -54,6 +54,8 @@ run reports the accepted spec the deploy response returned. | `1` | no compute named and none found in the project | | `1` | a selected unconfigured source directory has no explicit `--exposure` (`MissingComputeExposureError`) | | `1` | config records a runtime, size or exposure the CLI does not know | +| `1` | `[compute.] exclude` records a pattern the CLI cannot act on (`InvalidComputeExcludeError`) | +| `1` | `[compute.] exclude` matches every file in the source (`ComputeSourceMissingError`) | | `1` | a compute's source is missing, not a directory, or empty | | `1` | a compute's source directory cannot be read | | `1` | a compute's source links to a path outside itself | @@ -125,6 +127,49 @@ while `build_state` is `building`. The deploy response may carry an `image_version` — a re-push of a compute that is already serving echoes the image it is serving now — and that is the previous build's, not this one's. +## `[compute.] exclude` + +Patterns are read the way `.gitignore` reads them: one without `/` matches that name at any +depth, one with `/` is anchored at the compute's source directory, a trailing `/` matches +directories only, `**` spans directories, and within a single segment the syntax is the CLI's +existing glob matcher (`*`, `?`, `[a-z]`). An excluded directory is not descended into, so +everything beneath it is out too — which is also why the reported count counts the path the +walk turned back at rather than what sat underneath it. + +Two details follow from that pruning. A **trailing** `**` matches what is inside a directory +rather than the directory itself, so `cache/**` empties `cache` and leaves the directory in the +archive, while `cache` on its own removes it; a `**` anywhere else may still span nothing, so +`build/**/cache` matches `build/cache`. And repeated trailing separators are normalized before +anchoring is decided, so `dist/` and `dist//` are the same unanchored directory pattern. + +Re-inclusion (`!`) is **not** supported and is refused rather than read as a literal filename: +excluding a directory stops the walk there, so a pattern re-admitting something beneath it could +never be reached. An empty pattern, an empty path segment (`src//dist`), and a malformed +character class are refused the same way. Every refusal names the compute and the pattern. + +All of it is refused **before** anything is packaged or uploaded, beside the runtime, size and +exposure checks: a pattern the CLI cannot read is knowable from `config.toml` alone. + +`push` applies only the patterns the project recorded — there are no built-in defaults, so a +compute with no `exclude` key uploads its source directory whole, including one scaffolded by a +CLI that predates the setting. `compute new` is what writes a runtime's default list, into +`config.toml` where it can be read and edited. + +When patterns are configured, the packaged line carries the count: `Packaged +supabase/compute/api (3 files, 1.2 KiB, excluded 2 paths).` The clause is omitted entirely when +the compute records no patterns, so an unconfigured compute's output is unchanged. + +A source directory whose every file is excluded fails with `ComputeSourceMissingError` before +the upload, naming the patterns rather than reporting the "only empty directories" case that a +genuinely empty tree gets. The two are told apart by whether any path was actually excluded, +not by whether patterns were configured — a tree of nothing but empty directories packages to +zero files whatever `exclude` says, and blaming a pattern that matched nothing would send the +user to edit a line that is doing its job. + +An excluded symlink is excluded **before** it is vetted for escaping the build context, so +adding a hoisted `node_modules` link to `exclude` is a real answer to +`ComputeSourceEscapingLinkError` rather than something that failure pre-empts. + The presigned `PUT` above is the one request whose URL is itself a credential. `--debug` logs every request URL, so `httpClientLayer` redacts query strings that carry a signature. diff --git a/apps/cli/src/commands/experimental/compute/push/push.handler.ts b/apps/cli/src/commands/experimental/compute/push/push.handler.ts index 3f297e0e6e..e78827321d 100644 --- a/apps/cli/src/commands/experimental/compute/push/push.handler.ts +++ b/apps/cli/src/commands/experimental/compute/push/push.handler.ts @@ -17,6 +17,7 @@ import { formatBytes, packageComputeDirectory, } from "../../../../shared/compute/compute-package.ts"; +import { compileComputeExclude } from "../../../../shared/compute/compute-exclude.ts"; import { displayPath } from "../../../../shared/compute/compute-paths.ts"; import type { ComputeEntry } from "../../../../shared/compute/compute-config.ts"; import { @@ -319,26 +320,47 @@ const deployOneCompute = Effect.fnUntraced(function* (input: { override: input.exposure, }); + // Same reason, and the same place: a pattern the CLI cannot read is a config mistake, and + // finding it out mid-walk would mean reporting it after the packaging step announced itself. + const exclude = yield* compileComputeExclude({ name, patterns: compute.entry?.exclude }); + let contextUploadId: string; { const packaging = yield* output.task("Packaging compute..."); - const packaged = yield* packageComputeDirectory(compute.sourceDir).pipe( + const packaged = yield* packageComputeDirectory(compute.sourceDir, exclude).pipe( Effect.tapError(() => packaging.fail()), ); yield* packaging.clear(); + // The excluded count rides along on the same line, and only when patterns are configured: + // an over-broad pattern is otherwise visible only as a file count nobody had a number to + // compare against, and by then the archive is already uploaded. + const excludedNote = exclude.active + ? `, excluded ${packaged.excludedCount} ${packaged.excludedCount === 1 ? "path" : "paths"}` + : ""; yield* output.raw( `Packaged ${sourceDisplay} (${packaged.fileCount} files, ${formatBytes( packaged.archive.length, - )}).\n`, + )}${excludedNote}).\n`, "stderr", ); // The guard above only counts directory entries, so a tree of nothing but - // empty subdirectories still reaches here and packages to zero files. + // empty subdirectories still reaches here and packages to zero files — as + // does a directory whose every file the exclude patterns matched, which is + // the same outcome for a different reason and needs its own recovery. if (packaged.fileCount === 0) { + // Keyed on what the patterns actually removed rather than on whether any were + // configured: a tree of empty directories packages to zero files whatever `exclude` + // says, and blaming the patterns for it would send the user to edit a line that is + // doing nothing. + const excludedSomething = packaged.excludedCount > 0; return yield* new ComputeSourceMissingError({ - detail: `${sourceDisplay} holds no files to deploy, only empty directories.`, - suggestion: addYourCode(sourceDisplay), + detail: excludedSomething + ? `Every file in ${sourceDisplay} is matched by [compute.${name}] exclude, so there is nothing to deploy.` + : `${sourceDisplay} holds no files to deploy, only empty directories.`, + suggestion: excludedSomething + ? `Narrow [compute.${name}] exclude in supabase/config.toml so the files the build needs are packaged.` + : addYourCode(sourceDisplay), }); } diff --git a/apps/cli/src/commands/experimental/compute/push/push.integration.test.ts b/apps/cli/src/commands/experimental/compute/push/push.integration.test.ts index 9cc6252af7..2f161a7f28 100644 --- a/apps/cli/src/commands/experimental/compute/push/push.integration.test.ts +++ b/apps/cli/src/commands/experimental/compute/push/push.integration.test.ts @@ -24,6 +24,7 @@ import { ComputeSourceEscapingLinkError, ComputeSourceMissingError, ComputeUploadFailedError, + InvalidComputeExcludeError, MissingComputeExposureError, } from "../../../../shared/compute/compute.errors.ts"; import { computePush } from "./push.handler.ts"; @@ -2014,6 +2015,131 @@ describe("compute push", () => { }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), ); + describe("[compute.api] exclude", () => { + /** A project whose compute holds a secret and a dependency tree beside its entrypoint. */ + const withExtraFiles = (configExclude: string) => + project({ + "supabase/config.toml": `project_id = "demo"\n\n[compute.api]\nruntime = "node"\nsize = "2gb"\nexclude = ${configExclude}\n`, + "supabase/compute/api/.env": "SECRET=1\n", + "supabase/compute/api/node_modules/left-pad/index.js": "module.exports = 1;\n", + }); + + it.live("leaves the matched paths out of the uploaded context and says how many", () => + Effect.gen(function* () { + const repo = yield* withExtraFiles('[".env", "node_modules"]'); + const { layer, out } = setupCompute({ workdir: repo.dir, routes: routes() }); + + return yield* Effect.gen(function* () { + yield* push(); + + // Only `index.js` survives, and the two excluded paths are counted where the walk + // turned back — `node_modules/left-pad/index.js` is never reached to be counted. + expect(out.stderrText).toContain("1 files"); + expect(out.stderrText).toContain("excluded 2 paths"); + expect(out.stdoutText).toContain("Deployed Compute api"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + it.live("says nothing about exclusions when the compute configures none", () => + Effect.gen(function* () { + const repo = yield* project(); + const { layer, out } = setupCompute({ workdir: repo.dir, routes: routes() }); + + return yield* Effect.gen(function* () { + yield* push(); + + expect(out.stderrText).toContain("Packaged"); + expect(out.stderrText).not.toContain("excluded"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + // Refused where the refusal is still free: a pattern the CLI cannot read is knowable from + // config.toml alone, so nothing should have been packaged or uploaded to find it out. + it.live("refuses a re-inclusion pattern before anything is uploaded", () => + Effect.gen(function* () { + const repo = yield* withExtraFiles('["node_modules", "!node_modules/left-pad"]'); + const { layer, out, http } = setupCompute({ workdir: repo.dir, routes: routes() }); + + return yield* Effect.gen(function* () { + const error = yield* push().pipe(Effect.flip); + + expect(error).toBeInstanceOf(InvalidComputeExcludeError); + expect(http.requests).toHaveLength(0); + expect(out.stderrText).not.toContain("Packaged"); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + // An over-broad pattern leaves the same empty archive an empty directory would, and the + // recovery is the opposite one — narrow the patterns, not write more code. + it.live("names the patterns when they match every file in the source", () => + Effect.gen(function* () { + const repo = yield* withExtraFiles('["*"]'); + const { layer, http } = setupCompute({ workdir: repo.dir, routes: routes() }); + + return yield* Effect.gen(function* () { + const error = yield* push().pipe(Effect.flip); + + expect(error).toBeInstanceOf(ComputeSourceMissingError); + expect(error).toMatchObject({ + detail: expect.stringContaining("[compute.api] exclude"), + suggestion: expect.stringContaining("Narrow [compute.api] exclude"), + }); + // Nothing was uploaded, so the failure costs no remote state. + expect(http.requests).toHaveLength(0); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + // Configured patterns that matched nothing must not be blamed for an empty archive: the + // tree packages to zero files on its own, and the exclusion message would send the user + // to edit a line that is doing its job. + it.live("blames empty directories, not the patterns, when nothing was excluded", () => + Effect.gen(function* () { + const path = yield* Path.Path; + const fs = yield* FileSystem.FileSystem; + const repo = yield* withExtraFiles('["nothing-matches-this"]'); + for (const stale of [".env", "index.js"]) { + yield* fs.remove(path.join(repo.dir, "supabase", "compute", "api", stale)); + } + yield* fs.remove(path.join(repo.dir, "supabase", "compute", "api", "node_modules"), { + recursive: true, + }); + yield* fs.makeDirectory(path.join(repo.dir, "supabase", "compute", "api", "nested")); + const { layer, http } = setupCompute({ workdir: repo.dir, routes: routes() }); + + return yield* Effect.gen(function* () { + const error = yield* push().pipe(Effect.flip); + + expect(error).toBeInstanceOf(ComputeSourceMissingError); + expect(error).toMatchObject({ + detail: expect.stringContaining("only empty directories"), + suggestion: expect.stringContaining("Add your compute's code"), + }); + expect(http.requests).toHaveLength(0); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + + // A pattern the config schema accepts but no reader can act on: the loader lets it + // through as a list of strings, so this is `push`'s refusal, not a config-load failure. + it.live("refuses a malformed pattern", () => + Effect.gen(function* () { + const repo = yield* withExtraFiles('["src/[oops"]'); + const { layer, http } = setupCompute({ workdir: repo.dir, routes: routes() }); + + return yield* Effect.gen(function* () { + const error = yield* push().pipe(Effect.flip); + + expect(error).toBeInstanceOf(InvalidComputeExcludeError); + expect(http.requests).toHaveLength(0); + }).pipe(Effect.provide(layer)); + }).pipe(Effect.scoped, Effect.provide(BunServices.layer)), + ); + }); + // A malformed config.toml must fail inside the finalizers, or the run skips the // telemetry flush every invocation is supposed to perform. it.live("flushes telemetry when the project config cannot be loaded", () => diff --git a/apps/cli/src/shared/compute/compute-config.ts b/apps/cli/src/shared/compute/compute-config.ts index 861ffa84a2..9e4257d07e 100644 --- a/apps/cli/src/shared/compute/compute-config.ts +++ b/apps/cli/src/shared/compute/compute-config.ts @@ -5,7 +5,12 @@ import { type CliErrorActionabilityDeclaration, ErrorActionabilityId, } from "../telemetry/error-actionability.ts"; -import { appendTomlSection, isRenderableTomlNumber, tomlKey } from "./toml-section.ts"; +import { + appendTomlSection, + isRenderableTomlNumber, + tomlKey, + type TomlSectionValue, +} from "./toml-section.ts"; /** * The `[compute]` section of `supabase/config.toml`, read through the decoded @@ -23,6 +28,7 @@ export interface ComputeEntry { readonly exposure?: string; readonly instances?: number; readonly source?: string; + readonly exclude?: ReadonlyArray; } export interface ComputeSection { @@ -90,6 +96,20 @@ const isRecord = (value: unknown): value is Record => const instanceCountOrUndefined = (value: unknown): number | undefined => typeof value === "number" && Number.isInteger(value) && value >= 0 ? value : undefined; +/** + * The recorded exclude patterns, or `undefined` when the key is absent or holds something + * other than a list. Entries are left exactly as written, blanks included: `push` compiles + * them and names the one it cannot read, and a pattern dropped here would upload the file it + * was written to withhold. + */ +const patternsOrUndefined = (value: unknown): ReadonlyArray | undefined => { + if (!Array.isArray(value)) { + return undefined; + } + const patterns = value.filter((entry): entry is string => typeof entry === "string"); + return patterns.length === value.length ? patterns : undefined; +}; + /** * The decoded `[compute]` section as per-compute tables. Anything that is not an * object is dropped rather than read as a compute named after it. @@ -118,6 +138,7 @@ export function readComputeSection(compute: unknown): ComputeSection { exposure: recordedStringOrUndefined(value["exposure"]), instances: instanceCountOrUndefined(value["instances"]), source: stringOrUndefined(value["source"]), + exclude: patternsOrUndefined(value["exclude"]), }; } @@ -139,8 +160,8 @@ export interface ComputeEntryWrite { export const planComputeEntry = Effect.fnUntraced(function* (options: { readonly configPath: string; readonly name: string; - /** Rendered as written: strings are quoted, numbers are not. */ - readonly patch: Readonly>; + /** Rendered as written: strings are quoted, numbers are not, lists become TOML arrays. */ + readonly patch: Readonly>; /** The already-parsed config — the authority on whether an entry exists. */ readonly existingCompute: Readonly>; }) { diff --git a/apps/cli/src/shared/compute/compute-config.unit.test.ts b/apps/cli/src/shared/compute/compute-config.unit.test.ts index 11b78e3bbc..a3d63065d8 100644 --- a/apps/cli/src/shared/compute/compute-config.unit.test.ts +++ b/apps/cli/src/shared/compute/compute-config.unit.test.ts @@ -20,6 +20,7 @@ describe("readComputeSection", () => { exposure: "private", instances: 4, source: "packages/api", + exclude: ["node_modules", ".env"], }, box: { runtime: "sandbox" }, }), @@ -31,6 +32,7 @@ describe("readComputeSection", () => { exposure: "private", instances: 4, source: "packages/api", + exclude: ["node_modules", ".env"], }, box: { runtime: "sandbox", @@ -38,11 +40,27 @@ describe("readComputeSection", () => { exposure: undefined, instances: undefined, source: undefined, + exclude: undefined, }, }, }); }); + // Blanks survive so `push` can refuse the pattern by name; dropping one here would upload + // the file it was written to withhold. + test("keeps every recorded exclude pattern, blanks included", () => { + expect(readComputeSection({ api: { exclude: ["", "dist/"] } }).compute["api"]?.exclude).toEqual( + ["", "dist/"], + ); + }); + + test.each([ + ["a bare string", "node_modules"], + ["a list holding a non-string", ["dist/", 7]], + ])("reads %s as no exclude list at all", (_label, exclude) => { + expect(readComputeSection({ api: { exclude } }).compute["api"]?.exclude).toBeUndefined(); + }); + test("drops non-object values so a stray scalar is not read as a compute", () => { expect(readComputeSection({ stray: "oops", api: {} })).toEqual({ compute: { diff --git a/apps/cli/src/shared/compute/compute-exclude.ts b/apps/cli/src/shared/compute/compute-exclude.ts new file mode 100644 index 0000000000..7c6f955ee6 --- /dev/null +++ b/apps/cli/src/shared/compute/compute-exclude.ts @@ -0,0 +1,154 @@ +import { Effect } from "effect"; +import { pathMatch } from "../../command-internal/path-match.ts"; +import { InvalidComputeExcludeError } from "./compute.errors.ts"; + +/** + * `[compute.] exclude` — the patterns that keep a path out of the uploaded build + * context, read the way `.gitignore` reads them, with one segment matched by the CLI's own + * glob matcher ({@link pathMatch}). + * + * @see `../../commands/experimental/compute/push/SIDE_EFFECTS.md` — full semantics, the + * refusals, and why re-inclusion (`!`) is not offered. + */ + +interface ExcludePattern { + readonly raw: string; + /** Matched against the whole relative path rather than a single name. */ + readonly anchored: boolean; + /** Written with a trailing `/`, so it passes over a file of the same name. */ + readonly directoryOnly: boolean; + readonly segments: ReadonlyArray; +} + +export interface ComputeExcludeMatcher { + /** Whether any pattern is in play, so callers can skip reporting a count of zero. */ + readonly active: boolean; + readonly excludes: (relativePath: string, isDirectory: boolean) => boolean; +} + +/** Excludes nothing — a compute that records no patterns, and the default. */ +export const NO_COMPUTE_EXCLUSIONS: ComputeExcludeMatcher = { + active: false, + excludes: () => false, +}; + +/** Only a whole segment spans directories; `a**b` is the single-segment `a*b`. */ +const SPANNER = "**"; + +/** `pathMatch` reaches every operator in a pattern even once a match has failed. */ +function isWellFormedSegment(segment: string): boolean { + return !pathMatch(segment, "").badPattern; +} + +/** Adjacent spanners span exactly what one spans; left in, each retries the same suffixes. */ +function collapseSpanners(segments: ReadonlyArray): Array { + return segments.filter( + (segment, index) => segment !== SPANNER || segments[index - 1] !== SPANNER, + ); +} + +/** + * Matches pattern segments against path segments. An anchored pattern consumes the path + * entirely, since a matched directory is never descended into. + */ +function matchSegments(pattern: ReadonlyArray, segments: ReadonlyArray): boolean { + if (pattern.length === 0) { + return segments.length === 0; + } + const [head, ...rest] = pattern; + if (head === SPANNER) { + // A trailing `**` names what is inside a directory, so it must consume a segment or + // pruning would take the directory too; elsewhere a spanner may span nothing. + const fewest = rest.length === 0 ? 1 : 0; + for (let skipped = fewest; skipped <= segments.length; skipped++) { + if (matchSegments(rest, segments.slice(skipped))) { + return true; + } + } + return false; + } + const [first, ...remaining] = segments; + if (first === undefined) { + return false; + } + return pathMatch(head ?? "", first).matched && matchSegments(rest, remaining); +} + +/** + * Reads the recorded patterns into a matcher, naming any the CLI cannot act on. A pattern read + * as something else, or skipped, would upload the file it was written to withhold. + */ +export const compileComputeExclude = Effect.fnUntraced(function* (options: { + readonly name: string; + readonly patterns: ReadonlyArray | undefined; +}) { + const recorded = options.patterns; + if (recorded === undefined || recorded.length === 0) { + return NO_COMPUTE_EXCLUSIONS; + } + + const refuse = (raw: string, why: string, suggestion: string) => + new InvalidComputeExcludeError({ + detail: `[compute.${options.name}] exclude pattern "${raw}" ${why}.`, + suggestion, + }); + + const patterns: Array = []; + for (const raw of recorded) { + if (raw.startsWith("!")) { + return yield* refuse( + raw, + "re-includes a path, which the compute build context does not support", + `Drop the pattern, or narrow the pattern it was meant to carve out of under [compute.${options.name}] exclude.`, + ); + } + + // Anchoring is decided after trailing separators come off, so `dist/` and `dist//` agree. + const withoutTrailing = raw.replace(/\/+$/, ""); + const directoryOnly = withoutTrailing !== raw; + const anchored = withoutTrailing.includes("/"); + const body = withoutTrailing.replace(/^\/+/, ""); + + if (body === "") { + return yield* refuse( + raw, + "names no path", + `Remove it from [compute.${options.name}] exclude, or replace it with the path to leave out, for example "node_modules".`, + ); + } + + const segments = body.split("/"); + const malformed = segments.find( + (segment) => segment !== SPANNER && (segment === "" || !isWellFormedSegment(segment)), + ); + if (malformed !== undefined) { + return yield* refuse( + raw, + segments.includes("") + ? "has an empty path segment" + : // One verdict covers every malformed operator, so the segment is named, not a cause. + `has malformed glob syntax in "${malformed}"`, + `Fix the pattern under [compute.${options.name}] exclude, or replace it with a plain path such as "node_modules".`, + ); + } + + patterns.push({ raw, anchored, directoryOnly, segments: collapseSpanners(segments) }); + } + + return { + active: true, + excludes: (relativePath, isDirectory) => { + const segments = relativePath.split("/"); + const name = segments[segments.length - 1] ?? ""; + return patterns.some((pattern) => { + if (pattern.directoryOnly && !isDirectory) { + return false; + } + // An unanchored pattern is a single segment, so the name answers it at any depth. + return pattern.anchored + ? matchSegments(pattern.segments, segments) + : pathMatch(pattern.segments[0] ?? "", name).matched; + }); + }, + } satisfies ComputeExcludeMatcher; +}); diff --git a/apps/cli/src/shared/compute/compute-exclude.unit.test.ts b/apps/cli/src/shared/compute/compute-exclude.unit.test.ts new file mode 100644 index 0000000000..d784b154b3 --- /dev/null +++ b/apps/cli/src/shared/compute/compute-exclude.unit.test.ts @@ -0,0 +1,257 @@ +import { it } from "@effect/vitest"; +import { Effect } from "effect"; +import { describe, expect } from "vitest"; +import { compileComputeExclude, NO_COMPUTE_EXCLUSIONS } from "./compute-exclude.ts"; +import { InvalidComputeExcludeError } from "./compute.errors.ts"; +import { COMPUTE_RUNTIME_EXCLUSIONS, COMPUTE_RUNTIMES } from "./compute-runtimes.ts"; + +const compile = (patterns: ReadonlyArray) => + Effect.runSync(compileComputeExclude({ name: "api", patterns })); + +/** The refusal a pattern list earns, so each case can assert on the sentence the user reads. */ +const refusal = (patterns: ReadonlyArray) => + Effect.runSync(compileComputeExclude({ name: "api", patterns }).pipe(Effect.flip)).detail; + +describe("compileComputeExclude", () => { + it.effect("excludes nothing when the compute records no patterns", () => + Effect.gen(function* () { + const absent = yield* compileComputeExclude({ name: "api", patterns: undefined }); + const empty = yield* compileComputeExclude({ name: "api", patterns: [] }); + + expect(absent).toBe(NO_COMPUTE_EXCLUSIONS); + expect(empty).toBe(NO_COMPUTE_EXCLUSIONS); + expect(absent.active).toBe(false); + expect(absent.excludes("node_modules", true)).toBe(false); + }), + ); + + it.effect("reports itself active once a pattern is recorded", () => + Effect.gen(function* () { + const matcher = yield* compileComputeExclude({ name: "api", patterns: [".env"] }); + + expect(matcher.active).toBe(true); + }), + ); + + describe("a pattern with no separator matches that name at any depth", () => { + const matcher = compile(["node_modules"]); + + it.effect("matches at the top level", () => + Effect.sync(() => { + expect(matcher.excludes("node_modules", true)).toBe(true); + }), + ); + + it.effect("matches nested", () => + Effect.sync(() => { + expect(matcher.excludes("packages/api/node_modules", true)).toBe(true); + }), + ); + + it.effect("matches a file of the same name", () => + Effect.sync(() => { + expect(matcher.excludes("node_modules", false)).toBe(true); + }), + ); + + it.effect("leaves a name it merely prefixes alone", () => + Effect.sync(() => { + expect(matcher.excludes("node_modules.bak", true)).toBe(false); + }), + ); + }); + + describe("a pattern with a separator is anchored at the source directory", () => { + const matcher = compile(["/coverage", "src/*.test.ts"]); + + it.effect("matches at the root it is anchored to", () => + Effect.sync(() => { + expect(matcher.excludes("coverage", true)).toBe(true); + expect(matcher.excludes("src/index.test.ts", false)).toBe(true); + }), + ); + + it.effect("does not match the same name deeper in the tree", () => + Effect.sync(() => { + expect(matcher.excludes("packages/api/coverage", true)).toBe(false); + expect(matcher.excludes("app/src/index.test.ts", false)).toBe(false); + }), + ); + + it.effect("keeps a single wildcard inside one path segment", () => + Effect.sync(() => { + expect(matcher.excludes("src/nested/index.test.ts", false)).toBe(false); + }), + ); + }); + + describe("a trailing separator matches directories only", () => { + const matcher = compile(["dist/"]); + + // Anchoring is decided after the trailing separators come off, so a doubled one does not + // quietly turn a match-at-any-depth pattern into a root-only one. + it.effect("reads a doubled trailing separator the same way", () => + Effect.sync(() => { + const doubled = compile(["dist//"]); + + expect(doubled.excludes("dist", true)).toBe(true); + expect(doubled.excludes("packages/api/dist", true)).toBe(true); + expect(doubled.excludes("dist", false)).toBe(false); + }), + ); + + it.effect("matches the directory", () => + Effect.sync(() => { + expect(matcher.excludes("dist", true)).toBe(true); + expect(matcher.excludes("packages/api/dist", true)).toBe(true); + }), + ); + + it.effect("passes over a file of the same name", () => + Effect.sync(() => { + expect(matcher.excludes("dist", false)).toBe(false); + }), + ); + }); + + describe("`**` spans directories", () => { + const matcher = compile(["**/*.log", "build/**/cache"]); + + it.effect("matches at every depth, including none", () => + Effect.sync(() => { + expect(matcher.excludes("server.log", false)).toBe(true); + expect(matcher.excludes("a/b/c/server.log", false)).toBe(true); + }), + ); + + it.effect("spans zero or more segments between two fixed ones", () => + Effect.sync(() => { + expect(matcher.excludes("build/cache", true)).toBe(true); + expect(matcher.excludes("build/x/y/cache", true)).toBe(true); + }), + ); + + it.effect("still requires the segments around it to match", () => + Effect.sync(() => { + expect(matcher.excludes("build/x/cache/keep", false)).toBe(false); + expect(matcher.excludes("other/cache", true)).toBe(false); + }), + ); + + // Pruning makes this the difference between emptying a directory and deleting it: a + // trailing `**` that matched the parent would take the directory out of the archive too. + it.effect("empties a directory without removing it when trailing", () => + Effect.sync(() => { + const trailing = compile(["cache/**"]); + + expect(trailing.excludes("cache", true)).toBe(false); + expect(trailing.excludes("cache/blob", false)).toBe(true); + expect(trailing.excludes("cache/deep/blob", false)).toBe(true); + }), + ); + + // Repeats span exactly what one spanner spans, so they are folded rather than each + // retrying the same suffixes — the pathological case is a pattern, not an input path. + it.effect("treats repeated spanners as one", () => + Effect.sync(() => { + const repeated = compile(["**/**/**/**/missing"]); + + expect(repeated.excludes("a/b/c/d/e/f/g/h/missing", false)).toBe(true); + expect(repeated.excludes("a/b/c/d/e/f/g/h/present", false)).toBe(false); + }), + ); + }); + + it.effect("matches a character class within one segment", () => + Effect.sync(() => { + const matcher = compile(["*.[oa]"]); + + expect(matcher.excludes("main.o", false)).toBe(true); + expect(matcher.excludes("lib.a", false)).toBe(true); + expect(matcher.excludes("main.c", false)).toBe(false); + }), + ); + + it.effect("refuses a re-inclusion pattern rather than reading it as a filename", () => + Effect.sync(() => { + expect(refusal(["node_modules", "!node_modules/keep"])).toContain("re-includes a path"); + }), + ); + + it.effect("refuses a pattern that names no path", () => + Effect.sync(() => { + expect(refusal([""])).toContain("names no path"); + expect(refusal(["/"])).toContain("names no path"); + }), + ); + + // `pathMatch` reports one verdict for every malformed operator, so the message names the + // segment rather than claiming which operator broke. + it.effect.each([ + { label: "an unterminated character class", pattern: "src/[oops" }, + { label: "a trailing escape", pattern: "src/oops\\" }, + ])("refuses $label", ({ pattern }) => + Effect.sync(() => { + expect(refusal([pattern])).toContain("malformed glob syntax"); + }), + ); + + it.effect("refuses a pattern with an empty path segment", () => + Effect.sync(() => { + expect(refusal(["src//dist"])).toContain("empty path segment"); + }), + ); + + it.effect("names the compute and the pattern in every refusal", () => + Effect.gen(function* () { + const error = yield* compileComputeExclude({ + name: "worker", + patterns: ["!keep"], + }).pipe(Effect.flip); + + expect(error).toBeInstanceOf(InvalidComputeExcludeError); + expect(error.detail).toContain("[compute.worker] exclude"); + expect(error.detail).toContain('"!keep"'); + expect(error.suggestion).toContain("[compute.worker] exclude"); + }), + ); +}); + +describe("COMPUTE_RUNTIME_EXCLUSIONS", () => { + it.effect("records a compilable list for every offered runtime", () => + Effect.forEach(COMPUTE_RUNTIMES, (runtime) => + compileComputeExclude({ name: "api", patterns: COMPUTE_RUNTIME_EXCLUSIONS[runtime] }), + ), + ); + + it.effect("keeps environment files out of every runtime's build context", () => + Effect.sync(() => { + for (const runtime of COMPUTE_RUNTIMES) { + const matcher = compile(COMPUTE_RUNTIME_EXCLUSIONS[runtime]); + + expect(matcher.excludes(".env", false)).toBe(true); + expect(matcher.excludes(".env.production", false)).toBe(true); + } + }), + ); + + it.effect("keeps version-control metadata out of a worktree checkout too", () => + Effect.sync(() => { + for (const runtime of COMPUTE_RUNTIMES) { + const matcher = compile(COMPUTE_RUNTIME_EXCLUSIONS[runtime]); + + expect(matcher.excludes(".git", true)).toBe(true); + // A worktree or submodule has `.git` as a file holding the real gitdir path. + expect(matcher.excludes(".git", false)).toBe(true); + } + }), + ); + + it.effect("drops the node runtime's installed tree", () => + Effect.sync(() => { + const matcher = compile(COMPUTE_RUNTIME_EXCLUSIONS.node); + + expect(matcher.excludes("node_modules", true)).toBe(true); + }), + ); +}); diff --git a/apps/cli/src/shared/compute/compute-package.ts b/apps/cli/src/shared/compute/compute-package.ts index d7b1d6f976..1cdab9f5db 100644 --- a/apps/cli/src/shared/compute/compute-package.ts +++ b/apps/cli/src/shared/compute/compute-package.ts @@ -2,6 +2,7 @@ import { gzipSync } from "node:zlib"; import { Data, Effect, FileSystem, Option, Path } from "effect"; import type { PlatformError } from "effect/PlatformError"; import { ComputeSourceEscapingLinkError } from "./compute.errors.ts"; +import { type ComputeExcludeMatcher, NO_COMPUTE_EXCLUSIONS } from "./compute-exclude.ts"; import { createTar, type TarEntry } from "./tar.ts"; import { actionability, @@ -22,16 +23,21 @@ export class ComputeArchiveCompressionError extends Data.TaggedError( /** * Packages a compute's source directory into the `.tar.gz` build context the Compute API's - * upload slot expects. Nothing is excluded: for a `dockerfile` compute the archive is the - * build context the user's own `Dockerfile` expects, and for a catalog runtime the server - * synthesizes `FROM ` + `COPY` with no install step, so `node_modules/` is a deploy - * dependency rather than noise. Packaged size is reported back so growth is visible before - * the upload rather than after. + * upload slot expects. Nothing is excluded unless the project asks: for a `dockerfile` compute + * the archive is the build context the user's own `Dockerfile` expects, and a catalog runtime's + * build reads the same tree, so what belongs in it is the project's call rather than this + * packager's. `[compute.] exclude` is where that call is recorded. Packaged size is + * reported back so growth is visible before the upload rather than after. */ interface PackagedCompute { readonly archive: Uint8Array; readonly fileCount: number; + /** + * Paths the exclude patterns kept out, counted where the walk turned back rather than by + * what sat underneath — an excluded directory is one path, however much it held. + */ + readonly excludedCount: number; } /** @@ -72,18 +78,25 @@ function tarMtime(modified: Option.Option): number { return Number.isSafeInteger(seconds) && seconds > 0 ? seconds : 0; } +/** What one directory contributed: its tar entries, and how many paths `exclude` turned back. */ +interface CollectedEntries { + readonly entries: Array; + readonly excludedCount: number; +} + /** - * Every entry under `root`, as tar entries. Filesystem errors propagate rather than being - * skipped: an entry missing from the archive means deploying an application with a hole in - * it, reported as a success — an unreadable directory, an unopenable file, or an entry that - * vanishes mid-walk are all that case. + * Every entry under `root`, as tar entries, minus whatever `exclude` matches. Filesystem + * errors propagate rather than being skipped: an entry missing from the archive means + * deploying an application with a hole in it, reported as a success — an unreadable directory, + * an unopenable file, or an entry that vanishes mid-walk are all that case. */ const collectEntries = ( path: Path.Path, root: string, relativeDir: string, + exclude: ComputeExcludeMatcher, ): Effect.Effect< - Array, + CollectedEntries, PlatformError | ComputeSourceEscapingLinkError, FileSystem.FileSystem > => @@ -93,6 +106,7 @@ const collectEntries = ( const names = yield* fs.readDirectory(absoluteDir); const entries: Array = []; + let excludedCount = 0; for (const name of [...names].sort()) { const relativePath = relativeDir === "" ? name : `${relativeDir}/${name}`; @@ -104,6 +118,14 @@ const collectEntries = ( // from vanishing, and stops a link pointing at an ancestor from being walked into. const linkTarget = yield* fs.readLink(absolutePath).pipe(Effect.option); if (Option.isSome(linkTarget)) { + // Excluded before the confinement check below, not after: a hoisted `node_modules` + // link is the usual reason that check fails, so excluding it has to be the answer + // rather than something the failure pre-empts. A link is not itself a directory, so + // a `dir/` pattern passes over one, matching how `.gitignore` reads the same file. + if (exclude.excludes(relativePath, false)) { + excludedCount++; + continue; + } const confined = confinedLinkTarget({ path, root, @@ -114,7 +136,7 @@ const collectEntries = ( return yield* new ComputeSourceEscapingLinkError({ detail: `${relativePath} links to ${linkTarget.value}, which is outside the compute source and cannot be packaged with it.`, suggestion: - "Install the compute's dependencies inside its own directory, or point `source` at a directory that contains everything the build needs.", + "Add it to the compute's `exclude` patterns, or point `source` at a directory that contains everything the build needs.", }); } entries.push({ @@ -129,11 +151,18 @@ const collectEntries = ( const info = yield* fs.stat(absolutePath); + if (exclude.excludes(relativePath, info.type === "Directory")) { + excludedCount++; + continue; + } + const mtime = tarMtime(info.mtime); if (info.type === "Directory") { entries.push({ path: `${relativePath}/`, contents: new Uint8Array(0), mode: 0o755, mtime }); - entries.push(...(yield* collectEntries(path, root, relativePath))); + const nested = yield* collectEntries(path, root, relativePath, exclude); + entries.push(...nested.entries); + excludedCount += nested.excludedCount; continue; } @@ -155,14 +184,17 @@ const collectEntries = ( }); } - return entries; + return { entries, excludedCount } satisfies CollectedEntries; }); -export const packageComputeDirectory = Effect.fnUntraced(function* (dir: string) { +export const packageComputeDirectory = Effect.fnUntraced(function* ( + dir: string, + exclude: ComputeExcludeMatcher = NO_COMPUTE_EXCLUSIONS, +) { const path = yield* Path.Path; - const entries = yield* collectEntries(path, dir, ""); + const collected = yield* collectEntries(path, dir, "", exclude); - const tar = yield* createTar(entries); + const tar = yield* createTar(collected.entries); const archive = yield* Effect.try({ try: () => gzipSync(tar), catch: (cause) => @@ -174,7 +206,8 @@ export const packageComputeDirectory = Effect.fnUntraced(function* (dir: string) return { archive: new Uint8Array(archive), - fileCount: entries.filter((entry) => !entry.path.endsWith("/")).length, + fileCount: collected.entries.filter((entry) => !entry.path.endsWith("/")).length, + excludedCount: collected.excludedCount, } satisfies PackagedCompute; }); diff --git a/apps/cli/src/shared/compute/compute-package.unit.test.ts b/apps/cli/src/shared/compute/compute-package.unit.test.ts index 32b23aad1d..854d92c363 100644 --- a/apps/cli/src/shared/compute/compute-package.unit.test.ts +++ b/apps/cli/src/shared/compute/compute-package.unit.test.ts @@ -4,6 +4,7 @@ import { gunzipSync } from "node:zlib"; import { Cause, DateTime, Effect, Exit, FileSystem, Option, Path } from "effect"; import { describe, expect, test } from "vitest"; import { ComputeSourceEscapingLinkError } from "./compute.errors.ts"; +import { compileComputeExclude } from "./compute-exclude.ts"; import { formatBytes, packageComputeDirectory } from "./compute-package.ts"; import { TarFieldOutOfRangeError, TarPathTooLongError } from "./tar.ts"; @@ -142,6 +143,11 @@ describe("packageComputeDirectory", () => { expect(Option.isSome(failure) ? failure.value : undefined).toBeInstanceOf( ComputeSourceEscapingLinkError, ); + // Naming the escape hatch is the whole value of the refusal: excluding the link is + // cheaper than vendoring whatever it points at, and is the recovery `exclude` added. + expect(Option.isSome(failure) ? failure.value : undefined).toMatchObject({ + suggestion: expect.stringContaining("`exclude`"), + }); }), ), ); @@ -179,6 +185,111 @@ describe("packageComputeDirectory", () => { ), ); + describe("with [compute.] exclude patterns", () => { + const packExcluding = (root: string, patterns: ReadonlyArray) => + Effect.gen(function* () { + const exclude = yield* compileComputeExclude({ name: "api", patterns }); + return yield* packageComputeDirectory(root, exclude); + }).pipe(Effect.provide(BunServices.layer)); + + it.live("leaves a matched file out of the archive", () => + withTemp("supabase-compute-package-", (dir, fs, path) => + Effect.gen(function* () { + yield* fs.writeFileString(path.join(dir, "index.js"), "x"); + yield* fs.writeFileString(path.join(dir, ".env"), "SECRET=1"); + + const result = yield* packExcluding(dir, [".env"]); + + expect(readEntries(result.archive).map((entry) => entry.path)).toEqual(["index.js"]); + expect(result.fileCount).toBe(1); + expect(result.excludedCount).toBe(1); + }), + ), + ); + + // The walk turns back at the directory, so nothing underneath is read at all — which is + // the point for a dependency tree, and is also why the count is one rather than three. + it.live("leaves an excluded directory's whole subtree out, counted once", () => + withTemp("supabase-compute-package-", (dir, fs, path) => + Effect.gen(function* () { + yield* fs.writeFileString(path.join(dir, "index.js"), "x"); + yield* fs.makeDirectory(path.join(dir, "node_modules", "left-pad"), { recursive: true }); + yield* fs.writeFileString(path.join(dir, "node_modules", "left-pad", "index.js"), "p"); + yield* fs.writeFileString(path.join(dir, "node_modules", ".package-lock.json"), "{}"); + + const result = yield* packExcluding(dir, ["node_modules"]); + + expect(readEntries(result.archive).map((entry) => entry.path)).toEqual(["index.js"]); + expect(result.excludedCount).toBe(1); + }), + ), + ); + + it.live("matches an unanchored pattern at every depth", () => + withTemp("supabase-compute-package-", (dir, fs, path) => + Effect.gen(function* () { + yield* fs.makeDirectory(path.join(dir, "packages", "api"), { recursive: true }); + yield* fs.writeFileString(path.join(dir, "packages", "api", ".env"), "SECRET=1"); + yield* fs.writeFileString(path.join(dir, "packages", "api", "index.js"), "x"); + + const result = yield* packExcluding(dir, [".env"]); + + expect(readEntries(result.archive).map((entry) => entry.path)).toEqual([ + "packages/", + "packages/api/", + "packages/api/index.js", + ]); + }), + ), + ); + + it.live("keeps a file an anchored pattern only matches elsewhere", () => + withTemp("supabase-compute-package-", (dir, fs, path) => + Effect.gen(function* () { + yield* fs.makeDirectory(path.join(dir, "nested")); + yield* fs.writeFileString(path.join(dir, "keep.txt"), "k"); + yield* fs.writeFileString(path.join(dir, "nested", "keep.txt"), "n"); + + const result = yield* packExcluding(dir, ["/keep.txt"]); + + expect(readEntries(result.archive).map((entry) => entry.path)).toEqual([ + "nested/", + "nested/keep.txt", + ]); + }), + ), + ); + + // The link is excluded before it is vetted, so excluding it is a real answer to the + // hoisted-dependency failure rather than something that failure pre-empts. + it.live("excludes a symlink that would otherwise escape the build context", () => + withTemp("supabase-compute-package-", (dir, fs, path) => + Effect.gen(function* () { + yield* fs.writeFileString(path.join(dir, "index.js"), "x"); + yield* fs.symlink("../../elsewhere", path.join(dir, "node_modules")); + + const result = yield* packExcluding(dir, ["node_modules"]); + + expect(readEntries(result.archive).map((entry) => entry.path)).toEqual(["index.js"]); + expect(result.excludedCount).toBe(1); + }), + ), + ); + + it.live("reports nothing excluded when no pattern matches", () => + withTemp("supabase-compute-package-", (dir, fs, path) => + Effect.gen(function* () { + yield* fs.writeFileString(path.join(dir, "index.js"), "x"); + + const result = yield* packExcluding(dir, ["node_modules"]); + + expect(result.fileCount).toBe(1); + expect(result.excludedCount).toBe(0); + }), + ), + ); + }); + // A pre-1970 mtime is negative, and a negative number is not representable in // a USTAR octal field: `(-1).toString(8)` renders to exactly the field width, // so it would sail past the width check and ship a header GNU tar rejects diff --git a/apps/cli/src/shared/compute/compute-runtimes.ts b/apps/cli/src/shared/compute/compute-runtimes.ts index 5dd51a78ff..a901fb490f 100644 --- a/apps/cli/src/shared/compute/compute-runtimes.ts +++ b/apps/cli/src/shared/compute/compute-runtimes.ts @@ -41,6 +41,38 @@ export const COMPUTE_RUNTIME_DESCRIPTIONS: Record = { deno: "Deno catalog runtime (Web-standard fetch handler).", }; +/** + * The `[compute.] exclude` patterns `new` records for each runtime, read the way + * `.gitignore` reads them (see `./compute-exclude.ts`). + * + * Written into `config.toml` at scaffold time rather than applied silently at `push` time, so + * the list is visible, editable, and the same on every machine — a built-in default nobody + * could see would be a second, invisible source of truth for what ships. `push` therefore + * excludes nothing a project did not ask for, including for a compute scaffolded by an older + * CLI. + * + * Every runtime keeps environment files and version-control metadata out: both are secrets or + * noise in an image whose context is uploaded to the platform, whichever runtime builds it. + * `.git` carries no trailing slash because a worktree or submodule checkout has it as a file + * holding an absolute gitdir path, which a directory-only pattern would pass over. + * Beyond that the lists diverge by how each runtime resolves dependencies, so a runtime's own + * entry is the only place a pattern belongs. + */ +export const COMPUTE_RUNTIME_EXCLUSIONS: Record> = { + // The context is the user's own build context and their `Dockerfile` decides what it copies, + // so nothing beyond secrets and VCS metadata is assumed about its shape. + dockerfile: [".env", ".env.*", ".git"], + // The build resolves dependencies, so uploading a locally installed tree only ships this + // machine's platform-specific binaries. `*.log` covers the crash logs npm and yarn drop into + // the project root on a failed install. + node: [".env", ".env.*", ".git", "node_modules/", "*.log"], + // Deno resolves remote dependencies into a cache outside the project, so there is no + // installed tree here to drop; `node_modules/` appears only under an opt-in `nodeModulesDir`. + // `*.log` is kept for the same reason as above, since a Deno compute may still be installed + // from npm. + deno: [".env", ".env.*", ".git", "*.log"], +}; + /** * The only instance sizes offered, denominated by memory. There is no resize — a different size * means a new compute, not a `push` flag. diff --git a/apps/cli/src/shared/compute/compute.errors.ts b/apps/cli/src/shared/compute/compute.errors.ts index ae29737c74..702d34ecaf 100644 --- a/apps/cli/src/shared/compute/compute.errors.ts +++ b/apps/cli/src/shared/compute/compute.errors.ts @@ -38,11 +38,12 @@ export class MissingComputeNameError extends Data.TaggedError("MissingComputeNam /** * A symlink in the compute source points outside the build context. * - * The archive is everything the server gets, with no install step and no view of the surrounding - * repository, so a link whose target isn't also packaged arrives dangling — the catalog runtimes - * boot without the dependency, or a Dockerfile build fails on `COPY`, both minutes later with + * The archive is everything the server gets, with no view of the surrounding repository, so a + * link whose target isn't also packaged arrives dangling — the catalog runtimes boot without + * whatever it pointed at, or a Dockerfile build fails on `COPY`, both minutes later with * nothing naming the cause. Refused here instead. The common source is a package manager that - * hoists dependencies to the repository root, outside the compute's own `node_modules`. + * hoists dependencies to the repository root, outside the compute's own `node_modules`; adding + * the link to `[compute.] exclude` is the other way out. */ export class ComputeSourceEscapingLinkError extends Data.TaggedError( "ComputeSourceEscapingLinkError", @@ -55,6 +56,22 @@ export class ComputeSourceEscapingLinkError extends Data.TaggedError( } } +/** + * `[compute.] exclude` records a pattern the CLI cannot act on. + * + * Refused rather than skipped, because the setting exists to keep a specific file out of an + * archive that is uploaded and built: a pattern read as something other than what it says, or + * dropped, ships the secret or the stale dependency tree it was written to withhold. + */ +export class InvalidComputeExcludeError extends Data.TaggedError("InvalidComputeExcludeError")<{ + readonly detail: string; + readonly suggestion: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.invalidConfig; + } +} + /** A bare `push` found no compute to deploy — none named, none in the project. */ export class NoComputeToDeployError extends Data.TaggedError("NoComputeToDeployError")<{ readonly detail: string; diff --git a/apps/cli/src/shared/compute/toml-section.ts b/apps/cli/src/shared/compute/toml-section.ts index 2d49e15d6c..b33d57530b 100644 --- a/apps/cli/src/shared/compute/toml-section.ts +++ b/apps/cli/src/shared/compute/toml-section.ts @@ -58,15 +58,29 @@ export function tomlKey(key: string): string { return isBareKey(key) ? key : quote(key); } +/** A value a `[compute.]` key can be written as. */ +export type TomlSectionValue = string | number | ReadonlyArray; + /** - * `key = "value"`, or `key = value` for a number — quoting a count would write a TOML string, - * and the schema types `[compute.] instances` as a number, so a quoted count would stop - * `config.toml` from loading at all. Rendering doesn't validate: {@link isRenderableTomlNumber} - * is the guard that keeps `1.5`/`-1` from reaching here, since `planComputeEntry`'s re-parse is - * a syntax check, not a schema one. + * `key = "value"`, `key = value` for a number, or `key = ["a", "b"]` for a list — quoting a + * count would write a TOML string, and the schema types `[compute.] instances` as a + * number, so a quoted count would stop `config.toml` from loading at all. Rendering doesn't + * validate: {@link isRenderableTomlNumber} is the guard that keeps `1.5`/`-1` from reaching + * here, since `planComputeEntry`'s re-parse is a syntax check, not a schema one. + * + * A list renders on one line, however long: the appended table is read back and re-parsed + * before it is written, and a single-line array is the form that check is known to survive. */ -function renderPair(key: string, value: string | number): string { - return `${tomlKey(key)} = ${typeof value === "number" ? String(value) : quote(value)}`; +function renderPair(key: string, value: TomlSectionValue): string { + // Narrowed by what each branch is, not by what it isn't: `Array.isArray` does not narrow a + // `ReadonlyArray` out of the union, so testing for the array first left a cast behind. + const rendered = + typeof value === "number" + ? String(value) + : typeof value === "string" + ? quote(value) + : `[${value.map((entry) => quote(entry)).join(", ")}]`; + return `${tomlKey(key)} = ${rendered}`; } /** @@ -78,7 +92,7 @@ function renderPair(key: string, value: string | number): string { export function appendTomlSection( text: string, header: string, - values: Readonly>, + values: Readonly>, ): string { const block = [ `[${header}]`, diff --git a/apps/cli/src/shared/compute/toml-section.unit.test.ts b/apps/cli/src/shared/compute/toml-section.unit.test.ts index 73a1f2a901..1a36e29239 100644 --- a/apps/cli/src/shared/compute/toml-section.unit.test.ts +++ b/apps/cli/src/shared/compute/toml-section.unit.test.ts @@ -73,6 +73,24 @@ size = "2gb" ); }); + test("writes a list of patterns as a single-line TOML array", () => { + expect(appendTomlSection("", "compute.api", { exclude: [".env", "node_modules/*"] })).toBe( + '[compute.api]\nexclude = [".env", "node_modules/*"]\n', + ); + }); + + test("escapes a quote inside a list entry", () => { + expect(appendTomlSection("", "compute.api", { exclude: ['say"what'] })).toBe( + '[compute.api]\nexclude = ["say\\"what"]\n', + ); + }); + + test("writes an empty list rather than dropping the key", () => { + expect(appendTomlSection("", "compute.api", { exclude: [] })).toBe( + "[compute.api]\nexclude = []\n", + ); + }); + test("writes a header with no keys when there is nothing to set", () => { expect(appendTomlSection("", "compute.api", {})).toBe("[compute.api]\n"); }); diff --git a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt index ba6f6747c0..1a8bff1c6f 100644 --- a/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt +++ b/apps/cli/src/shared/telemetry/__fixtures__/error-tags.txt @@ -301,6 +301,7 @@ InspectMutuallyExclusiveFlagsError InspectReportMkdirError InspectReportWriteError InvalidAccessTokenError +InvalidComputeExcludeError InvalidComputeNameError InvalidComputeSourceError InvalidFunctionDeploySlugError diff --git a/packages/config/src/compute.ts b/packages/config/src/compute.ts index 034939a614..cd5dcb85a4 100644 --- a/packages/config/src/compute.ts +++ b/packages/config/src/compute.ts @@ -70,6 +70,24 @@ const computeEntry = Schema.Struct({ tags, }), ), + exclude: Schema.optionalKey( + // Patterns are left unvalidated here so that `push` can refuse a specific one by name + // instead of failing the whole config load: unlike `instances`, a pattern the CLI cannot + // read is never silently dropped, so there is nothing for this layer to protect. + Schema.Array(Schema.String).annotate({ + description: dedent` + Patterns for paths to leave out of the uploaded build context, read the way + \`.gitignore\` reads them: a pattern without \`/\` matches that name at any depth, + one with \`/\` is anchored at the compute's source directory, a trailing \`/\` + matches directories only, and \`**\` spans directories. Excluding a directory + excludes everything under it; re-inclusion (\`!\`) is not supported. Only the + patterns recorded here are excluded, so a compute with no list uploads its source + directory whole. + `, + examples: [["node_modules", ".env"]], + tags, + }), + ), }); /** `[compute]` — one `[compute.]` table per Compute service, keyed by name. */ diff --git a/packages/config/src/compute.unit.test.ts b/packages/config/src/compute.unit.test.ts index 2e74b71f43..e402d5b557 100644 --- a/packages/config/src/compute.unit.test.ts +++ b/packages/config/src/compute.unit.test.ts @@ -15,6 +15,7 @@ describe("compute schema", () => { exposure: "private", instances: 3, source: "packages/api", + exclude: ["node_modules", ".env"], }, }; expect(decode(every)).toEqual(every); @@ -42,6 +43,25 @@ describe("compute schema", () => { expect(decode({ api: {} })).toEqual({ api: {} }); }); + test("decodes exclude patterns in the order they were written", () => { + expect(decode({ api: { exclude: ["dist/", "**/*.log"] } })).toEqual({ + api: { exclude: ["dist/", "**/*.log"] }, + }); + }); + + test("accepts an empty exclude list as excluding nothing", () => { + expect(decode({ api: { exclude: [] } })).toEqual({ api: { exclude: [] } }); + }); + + // Patterns reach the CLI verbatim so `push` can name the one it cannot read; the schema's + // job is only to establish that the key holds a list of strings at all. + test.each([ + ["a bare string", "node_modules"], + ["a non-string entry", [1]], + ])("rejects %s as an exclude list", (_label, exclude) => { + expect(() => decode({ api: { exclude } })).toThrow(); + }); + test("rejects a non-numeric instance count", () => { expect(() => decode({ api: { instances: "three" } })).toThrow(); }); @@ -71,6 +91,16 @@ describe("compute schema", () => { expect(computeSchema?.properties?.exposure).toBeDefined(); expect(computeSchema?.properties?.instances).toBeDefined(); expect(computeSchema?.properties?.source).toBeDefined(); + expect(computeSchema?.properties?.exclude).toBeDefined(); + }); + + test("types exclude as an array of strings in the generated JSON schema", () => { + const json = JSON.parse(JSON.stringify(Schema.toJsonSchemaDocument(compute).schema)); + const objectSchema = json.anyOf?.find((entry: { type?: string }) => entry?.type === "object"); + const computeSchema = objectSchema?.patternProperties?.[computeNamePattern]; + + expect(computeSchema?.properties?.exclude?.type).toBe("array"); + expect(computeSchema?.properties?.exclude?.items?.type).toBe("string"); }); test("bounds instances as a non-negative integer in the generated JSON schema", () => {