From 76e731d7e7a650a327d31dfe6d8d4022a54803fc Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Mon, 14 Sep 2026 17:43:48 +0530 Subject: [PATCH 1/2] feat(cli): add a resource_limit error category for memory-limit kills Adds a resource_limit category under user_actionable and its out_of_memory / container_killed fingerprint suffixes, so an edge runtime container killed for exceeding its memory limit can be classified separately from an internal bug. Co-Authored-By: Claude Opus 5 --- .../shared/telemetry/error-actionability.ts | 19 +++++++- .../error-actionability.unit.test.ts | 44 +++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/apps/cli/src/shared/telemetry/error-actionability.ts b/apps/cli/src/shared/telemetry/error-actionability.ts index 725c351284..e4161ec40d 100644 --- a/apps/cli/src/shared/telemetry/error-actionability.ts +++ b/apps/cli/src/shared/telemetry/error-actionability.ts @@ -36,6 +36,7 @@ export const CliErrorCategory = { PlanLimit: "plan_limit", ProjectPaused: "project_paused", InvalidInput: "invalid_input", + ResourceLimit: "resource_limit", Network: "network", ApiStatus: "api_status", Cancelled: "cancelled", @@ -86,6 +87,7 @@ const CLI_ERROR_FINGERPRINT_SUFFIXES = [ "cancelled", "connect", "container_configuration", + "container_killed", "daemon_start", "daemon_protocol", "daemon_status", @@ -106,6 +108,7 @@ const CLI_ERROR_FINGERPRINT_SUFFIXES = [ "invalid_config", "network", "not_found", + "out_of_memory", "plan_limit", "platform_error", "port_allocation", @@ -132,7 +135,8 @@ type UserActionableErrorCategory = | typeof CliErrorCategory.Permission | typeof CliErrorCategory.PlanLimit | typeof CliErrorCategory.ProjectPaused - | typeof CliErrorCategory.InvalidInput; + | typeof CliErrorCategory.InvalidInput + | typeof CliErrorCategory.ResourceLimit; type CliErrorKindCategory = | { @@ -354,6 +358,16 @@ export const actionability = { suggestion_type: CliSuggestionType.RunCommand, suggested_command: "supabase stop", }, + /** + * A container was killed for exceeding its memory limit — the user can raise + * the runtime's memory allocation, so this is not an internal bug. + */ + resourceLimit: { + error_kind: CliErrorKind.UserActionable, + error_category: CliErrorCategory.ResourceLimit, + has_suggestion: true, + suggestion_type: CliSuggestionType.UpdateConfig, + }, externalNetwork: { error_kind: CliErrorKind.ExternalService, error_category: CliErrorCategory.Network, @@ -479,7 +493,8 @@ function isUserActionableCategory(value: unknown): value is UserActionableErrorC value === CliErrorCategory.Permission || value === CliErrorCategory.PlanLimit || value === CliErrorCategory.ProjectPaused || - value === CliErrorCategory.InvalidInput + value === CliErrorCategory.InvalidInput || + value === CliErrorCategory.ResourceLimit ); } diff --git a/apps/cli/src/shared/telemetry/error-actionability.unit.test.ts b/apps/cli/src/shared/telemetry/error-actionability.unit.test.ts index 50b950904d..b6d773b839 100644 --- a/apps/cli/src/shared/telemetry/error-actionability.unit.test.ts +++ b/apps/cli/src/shared/telemetry/error-actionability.unit.test.ts @@ -5,9 +5,12 @@ import { SupabaseApiInputError, markSupabaseApiInputErrorAsUserInput } from "@su import { BootstrapHealthError } from "../../commands/bootstrap/bootstrap.errors.ts"; import { actionability, + CliErrorCategory, type CliErrorActionabilityDeclaration, + CliErrorKind, classifyCliCauseActionability, classifyCliErrorActionability, + CliSuggestionType, ErrorActionabilityFingerprintId, ErrorActionabilityId, statusCodeActionability, @@ -47,6 +50,29 @@ class RuntimeCrashError extends Data.TaggedError("RuntimeCrashError")<{ } } +class ResourceLimitError extends Data.TaggedError("ResourceLimitError")<{ + readonly message: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return actionability.resourceLimit; + } +} + +// `resource_limit` is only valid paired with `user_actionable`; this pairing +// must be rejected so it does not silently count against `internal_bug`. +class MisclassifiedResourceLimitError extends Data.TaggedError("MisclassifiedResourceLimitError")<{ + readonly message: string; +}> { + get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + return { + error_kind: CliErrorKind.InternalBug, + error_category: CliErrorCategory.ResourceLimit, + has_suggestion: false, + suggestion_type: CliSuggestionType.None, + } as unknown as CliErrorActionabilityDeclaration; + } +} + function externalError( _tag: string, fields: Record = {}, @@ -221,6 +247,24 @@ describe("classifyCliErrorActionability", () => { suggestion_type: "rerun_debug", }); }); + + it("accepts user_actionable paired with resource_limit", () => { + expect(classifyCliErrorActionability(new ResourceLimitError({ message: "private" }))).toEqual({ + error_kind: "user_actionable", + error_category: "resource_limit", + error_fingerprint: "tag:ResourceLimitError", + has_suggestion: true, + suggestion_type: "update_config", + }); + }); + + it("rejects resource_limit paired with internal_bug rather than counting it as our bug", () => { + const result = classifyCliErrorActionability( + new MisclassifiedResourceLimitError({ message: "private" }), + ); + expect(result.error_kind).toBe("unknown"); + expect(result.error_category).toBe("unknown"); + }); }); describe("classifyCliCauseActionability", () => { From d322ccd912ad9d66fa932f7a3501443dce7c758f Mon Sep 17 00:00:00 2001 From: Prashansa Kulshrestha Date: Tue, 15 Sep 2026 10:33:53 +0530 Subject: [PATCH 2/2] fix(cli): stop retrying an edge runtime container that was killed Exit 137 re-attached to the log stream until the re-attach cap gave up, reporting an out-of-memory kill as a lost log stream against a container the message claimed was still running. `State.OOMKilled` separates a memory-limit kill, which carries a memory-allocation remediation, from a kill the CLI cannot attribute to either side. Co-Authored-By: Claude Opus 5 --- .../src/command-internal/docker-lifecycle.ts | 8 +- .../docker-lifecycle.unit.test.ts | 32 ++++++-- .../src/command-internal/docker-suggest.ts | 4 + .../commands/functions/serve/SIDE_EFFECTS.md | 26 +++--- .../functions/serve/serve.integration.test.ts | 81 ++++++++++++++++++- apps/cli/src/shared/functions/serve.errors.ts | 18 ++++- .../functions/serve.errors.unit.test.ts | 48 +++++++++++ apps/cli/src/shared/functions/serve.ts | 8 +- 8 files changed, 198 insertions(+), 27 deletions(-) diff --git a/apps/cli/src/command-internal/docker-lifecycle.ts b/apps/cli/src/command-internal/docker-lifecycle.ts index ae6e60c486..d2ad2ca18a 100644 --- a/apps/cli/src/command-internal/docker-lifecycle.ts +++ b/apps/cli/src/command-internal/docker-lifecycle.ts @@ -249,6 +249,7 @@ function parseContainerState(stdout: string): { readonly running: boolean; readonly status: string; readonly exitCode: number; + readonly oomKilled: boolean; readonly health?: string; } { const trimmed = stdout.trim(); @@ -265,12 +266,15 @@ function parseContainerState(stdout: string): { const status = typeof state["Status"] === "string" ? state["Status"] : ""; const running = state["Running"] === true; const exitCode = typeof state["ExitCode"] === "number" ? state["ExitCode"] : 0; + // Docker sets `OOMKilled` only when the container's own memory limit was hit; a host-level + // out-of-memory kill reports the same exit code with `OOMKilled: false`. + const oomKilled = state["OOMKilled"] === true; const health = state["Health"]; const healthStatus = isJsonRecord(health) && typeof health["Status"] === "string" ? health["Status"] : undefined; return healthStatus !== undefined - ? { running, status, exitCode, health: healthStatus } - : { running, status, exitCode }; + ? { running, status, exitCode, oomKilled, health: healthStatus } + : { running, status, exitCode, oomKilled }; } function isJsonRecord(value: unknown): value is { readonly [key: string]: unknown } { diff --git a/apps/cli/src/command-internal/docker-lifecycle.unit.test.ts b/apps/cli/src/command-internal/docker-lifecycle.unit.test.ts index ac4be04fb5..97165a31c4 100644 --- a/apps/cli/src/command-internal/docker-lifecycle.unit.test.ts +++ b/apps/cli/src/command-internal/docker-lifecycle.unit.test.ts @@ -230,6 +230,7 @@ describe("inspectContainerState", () => { running: true, status: "running", exitCode: 0, + oomKilled: false, health: "healthy", }); expect(mock.spawned).toEqual([ @@ -248,7 +249,28 @@ describe("inspectContainerState", () => { }); return inspectContainerState(mock.spawner, "supabase_kong_my-app").pipe( Effect.map((state) => { - expect(state).toEqual({ running: true, status: "running", exitCode: 0 }); + expect(state).toEqual({ running: true, status: "running", exitCode: 0, oomKilled: false }); + }), + ); + }); + + it.live("reports a container killed for exceeding its memory limit", () => { + const mock = mockSpawner({ + stdout: JSON.stringify({ + Status: "exited", + Running: false, + ExitCode: 137, + OOMKilled: true, + }), + }); + return inspectContainerState(mock.spawner, "supabase_edge_runtime_my-app").pipe( + Effect.map((state) => { + expect(state).toEqual({ + running: false, + status: "exited", + exitCode: 137, + oomKilled: true, + }); }), ); }); @@ -259,7 +281,7 @@ describe("inspectContainerState", () => { }); return inspectContainerState(mock.spawner, "supabase_kong_my-app").pipe( Effect.map((state) => { - expect(state).toEqual({ running: false, status: "exited", exitCode: 1 }); + expect(state).toEqual({ running: false, status: "exited", exitCode: 1, oomKilled: false }); }), ); }); @@ -272,7 +294,7 @@ describe("inspectContainerState", () => { }); return inspectContainerState(mock.spawner, "supabase_db_my-app").pipe( Effect.map((state) => { - expect(state).toEqual({ running: true, status: "paused", exitCode: 0 }); + expect(state).toEqual({ running: true, status: "paused", exitCode: 0, oomKilled: false }); }), ); }, @@ -334,7 +356,7 @@ describe("inspectContainerState", () => { const mock = mockSpawner({ stdout: "" }); return inspectContainerState(mock.spawner, "supabase_db_my-app").pipe( Effect.map((state) => { - expect(state).toEqual({ running: false, status: "", exitCode: 0 }); + expect(state).toEqual({ running: false, status: "", exitCode: 0, oomKilled: false }); }), ); }); @@ -343,7 +365,7 @@ describe("inspectContainerState", () => { const mock = mockSpawner({ stdout: "null" }); return inspectContainerState(mock.spawner, "supabase_db_my-app").pipe( Effect.map((state) => { - expect(state).toEqual({ running: false, status: "", exitCode: 0 }); + expect(state).toEqual({ running: false, status: "", exitCode: 0, oomKilled: false }); }), ); }); diff --git a/apps/cli/src/command-internal/docker-suggest.ts b/apps/cli/src/command-internal/docker-suggest.ts index 999c70febf..7690411e3f 100644 --- a/apps/cli/src/command-internal/docker-suggest.ts +++ b/apps/cli/src/command-internal/docker-suggest.ts @@ -8,6 +8,10 @@ export const SUGGEST_DOCKER_INSTALL = export const SUGGEST_DOCKER_START = "Docker is no longer reachable. Start Docker, then rerun `supabase functions serve`."; +/** Remediation hint shown when a container is killed for exceeding its memory limit. */ +export const SUGGEST_CONTAINER_MEMORY_LIMIT = + "The container was killed for exceeding its memory limit. Raise the memory allocated to your container runtime and try again."; + /** * Whether a container-CLI stderr indicates the daemon is unreachable. Matches the docker/podman * "cannot connect"/"is the docker daemon running" messages, a socket permission-denied message, diff --git a/apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md b/apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md index 9f7a2a9af4..7b0bf8d5e3 100644 --- a/apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md @@ -61,17 +61,19 @@ back to local keys. No scheme/host validation is performed on the discovered URL ## Exit Codes -| Code | Condition | -| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `0` | clean shutdown after `SIGINT` or `SIGTERM` | -| `0` | the edge-runtime container stops on its own with exit code `0` | -| `0` | the edge-runtime container is torn down by an external supervisor — exit `129`/`130`/`131`/`143` (`SIGHUP`/`SIGINT`/`SIGQUIT`/`SIGTERM`), e.g. `supabase stop` run in another terminal | -| `0` | the edge-runtime container is already gone by the time a follow-up `docker container inspect` runs after the log stream ended | -| `0` | an edge-runtime startup failure or log-stream failure lands within the shutdown grace period (~50ms) of a `SIGINT`/`SIGTERM` | -| `1` | local DB container is not running, or the Docker daemon is unreachable (surfaces from the DB inspect as `failed to inspect service: …` plus the Docker Desktop install suggestion) | -| `1` | invalid inspect flag combination, or a `Config.Validate` failure anywhere in `config.toml` (not just project/auth config) | -| `1` | env file, signing key, import map, or function bind resolution failure | -| `1` | edge-runtime container startup, log streaming, or restart loop failure — including the edge-runtime container crashing with any exit code other than `0`, `137`, or `129`/`130`/`131`/`143` | +| Code | Condition | +| ---- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `0` | clean shutdown after `SIGINT` or `SIGTERM` | +| `0` | the edge-runtime container stops on its own with exit code `0` | +| `0` | the edge-runtime container is torn down by an external supervisor — exit `129`/`130`/`131`/`143` (`SIGHUP`/`SIGINT`/`SIGQUIT`/`SIGTERM`), e.g. `supabase stop` run in another terminal | +| `0` | the edge-runtime container is already gone by the time a follow-up `docker container inspect` runs after the log stream ended | +| `0` | an edge-runtime startup failure or log-stream failure lands within the shutdown grace period (~50ms) of a `SIGINT`/`SIGTERM` | +| `1` | local DB container is not running, or the Docker daemon is unreachable (surfaces from the DB inspect as `failed to inspect service: …` plus the Docker Desktop install suggestion) | +| `1` | invalid inspect flag combination, or a `Config.Validate` failure anywhere in `config.toml` (not just project/auth config) | +| `1` | env file, signing key, import map, or function bind resolution failure | +| `1` | edge-runtime container startup, log streaming, or restart loop failure — including the edge-runtime container crashing with any exit code other than `0` or `129`/`130`/`131`/`143` | +| `1` | the edge-runtime container is killed for exceeding its memory limit — exit `137` with `State.OOMKilled` | +| `1` | the edge-runtime container is killed from outside the CLI — exit `137` without `State.OOMKilled` | ## Telemetry Events Fired @@ -121,7 +123,7 @@ Long-running raw log / error events only; there is no terminal `result` event on - Config, project dotenv discovery, and function discovery all resolve from `` with no ancestor search (CLI-2285), so they can never disagree. - Before each container (re)start, resolves the edge-runtime image through the same registry-candidate pull-with-retry every native `functions` Docker path uses: `docker image inspect ` (ECR, then GHCR, then Docker Hub) to check the local cache, then `docker pull ` with 2 retries (4s/8s backoff) on a miss, after `assertLocalDbRunning` — resolving it earlier would hijack the down-daemon error message that DB-inspect step is responsible for producing. - Runs the full `Config.Validate` pipeline (`resolveLocalConfigValues`, same one `start`/`stop`/`status` use) on every startup/restart, before `assertLocalDbRunning` — an invalid config now fails `serve` up front even for fields this command never otherwise reads (e.g. a bad `db.major_version` or malformed auth hook). -- A container that stops on its own with exit code `0`, or that is torn down by an external supervisor (exit `129`/`130`/`131`/`143`, e.g. `supabase stop` in another terminal), or that is already gone by the time a follow-up inspect runs, all end the command successfully — each prints its own distinct line (see Output above) rather than the user-initiated `Stopped serving …` line, so scrollback can tell "I stopped it" from "the runtime walked out" or "a supervisor tore it down". In a `functions serve &` CI step this means a runtime that exits on its own does not fail the step; the distinct message is the only signal, and a downstream failure otherwise only surfaces later as connection-refused. Exit `137` (SIGKILL, e.g. an OOM kill) is retried by re-attaching to the log stream rather than failing the command. Any other non-zero container exit fails the command; the error message includes the container id. Only a watched-file change restarts the container itself — none of these outcomes ever restart it. +- A container that stops on its own with exit code `0`, or that is torn down by an external supervisor (exit `129`/`130`/`131`/`143`, e.g. `supabase stop` in another terminal), or that is already gone by the time a follow-up inspect runs, all end the command successfully — each prints its own distinct line (see Output above) rather than the user-initiated `Stopped serving …` line, so scrollback can tell "I stopped it" from "the runtime walked out" or "a supervisor tore it down". In a `functions serve &` CI step this means a runtime that exits on its own does not fail the step; the distinct message is the only signal, and a downstream failure otherwise only surfaces later as connection-refused. Exit `137` (SIGKILL) fails the command: `State.OOMKilled` separates a container that hit its memory limit, which reports the memory-allocation remediation, from one killed by something the CLI cannot identify. Any other non-zero container exit fails the command; the error message includes the container id. Only a watched-file change restarts the container itself — none of these outcomes ever restart it. - A `docker logs -f` re-attach (the daemon can close the stream while the container keeps running) resumes with `--since ` instead of replaying the full log history, and is capped at 5 consecutive re-attaches that forward no new output; exceeding the cap fails the command with a tagged error instead of looping forever. - On the log-stream path, the Docker-daemon-unreachable classification comes from a follow-up `docker container inspect` failure, not from `docker logs -f`'s own stderr text. - The worker bootstrap template (`serve.main.ts`) is bundled into a single self-contained module with `jose` and the local path/status helpers inlined, so the edge-runtime worker boots without any network access (supabase/supabase#45570). The bundle is embedded at build time for shipped binaries and produced on demand (esbuild) when running from source. It is delivered into the created (not yet started) container as a `docker cp` stdin tar archive at `/root/index.ts` — never a single-file host bind mount, which materializes as an empty directory on daemons that cannot see the client's filesystem (remote `DOCKER_HOST`/Docker-context daemons, podman machines) and breaks bring-up with edge-runtime's "failed to determine entrypoint" (supabase/cli#6254). Only this bootstrap template is daemon-independent: user function sources, import maps, static files, and the multiline-env script directory (present only when an env value contains a newline) still arrive by host bind mounts, so they require a daemon that can see the project directory. diff --git a/apps/cli/src/commands/functions/serve/serve.integration.test.ts b/apps/cli/src/commands/functions/serve/serve.integration.test.ts index c2e0f4fdd9..d9100b9d96 100644 --- a/apps/cli/src/commands/functions/serve/serve.integration.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.integration.test.ts @@ -35,6 +35,7 @@ import { } from "../../../../tests/helpers/mocks.ts"; import { CommandSettings } from "../../../config/command-settings.service.ts"; import { functionsGoConfigCompat } from "../../../command-internal/functions-go-config.ts"; +import { SUGGEST_CONTAINER_MEMORY_LIMIT } from "../../../command-internal/docker-suggest.ts"; import { DebugFlag, NetworkIdFlag } from "../../../command-internal/global-flags.ts"; import { FileWatcher, type FileWatchEvent } from "../../../shared/runtime/file-watcher.service.ts"; import { @@ -2119,13 +2120,18 @@ describe("functions serve integration", () => { } // Models `inspectContainerState`'s `docker container inspect --format {{json .State}}` reply. - function inspectStateBehavior(running: boolean, exitCode = 0): LogProcessBehavior { + function inspectStateBehavior( + running: boolean, + exitCode = 0, + oomKilled = false, + ): LogProcessBehavior { return { exitCode: 0, stdout: JSON.stringify({ Status: running ? "running" : "exited", Running: running, ExitCode: exitCode, + OOMKilled: oomKilled, }), stderr: "", }; @@ -2344,6 +2350,79 @@ describe("functions serve integration", () => { }, ); + it.live( + "fails as an out-of-memory kill, without retrying, when the container is OOM-killed (exit 137)", + () => { + deployMockState.runHandler = baseDockerRunHandler(); + const childSpawner = mockDockerLogSpawner([ + { exitCode: 0 }, + inspectStateBehavior(false, 137, true), + ]); + + return Effect.gen(function* () { + yield* Effect.promise(writeHelloFunction); + + const { layer } = setupServe({ childSpawner }); + // A container killed for exceeding its memory limit never comes back, so a + // regression to re-attaching burns the whole cap before failing; bound the + // wait so that shows up as a timeout rather than a slow pass. + const error = yield* functionsServe(baseFlags()).pipe( + Effect.provide(layer), + Effect.timeout(Duration.seconds(5)), + Effect.flip, + ); + + expect(error).toBeInstanceOf(EdgeRuntimeContainerCrashedError); + if (error instanceof EdgeRuntimeContainerCrashedError) { + expect(error.exitCode).toBe(137); + expect(error.oomKilled).toBe(true); + expect(error.suggestion).toBe(SUGGEST_CONTAINER_MEMORY_LIMIT); + expect(error[ErrorActionabilityId]).toEqual({ + ...actionability.resourceLimit, + fingerprint_suffix: "out_of_memory", + }); + } + expect(containerInspectCalls(childSpawner)).toHaveLength(1); + }); + }, + ); + + it.live( + "fails as unattributable, without retrying, when the container is killed from outside the CLI (exit 137)", + () => { + deployMockState.runHandler = baseDockerRunHandler(); + const childSpawner = mockDockerLogSpawner([ + { exitCode: 0 }, + inspectStateBehavior(false, 137, false), + ]); + + return Effect.gen(function* () { + yield* Effect.promise(writeHelloFunction); + + const { layer } = setupServe({ childSpawner }); + const error = yield* functionsServe(baseFlags()).pipe( + Effect.provide(layer), + Effect.timeout(Duration.seconds(5)), + Effect.flip, + ); + + expect(error).toBeInstanceOf(EdgeRuntimeContainerCrashedError); + if (error instanceof EdgeRuntimeContainerCrashedError) { + expect(error.exitCode).toBe(137); + expect(error.oomKilled).toBe(false); + expect(error.suggestion).toBeUndefined(); + const declaration = error[ErrorActionabilityId]; + expect(declaration).toEqual({ + ...actionability.unknown, + fingerprint_suffix: "container_killed", + }); + expect(declaration.error_kind).not.toBe("internal_bug"); + } + expect(containerInspectCalls(childSpawner)).toHaveLength(1); + }); + }, + ); + it.live( "ends the session normally, with a distinct message, when the container exits gracefully (exit 0)", () => { diff --git a/apps/cli/src/shared/functions/serve.errors.ts b/apps/cli/src/shared/functions/serve.errors.ts index 79aae8e2bc..498aee6223 100644 --- a/apps/cli/src/shared/functions/serve.errors.ts +++ b/apps/cli/src/shared/functions/serve.errors.ts @@ -1,5 +1,6 @@ import { Data } from "effect"; import { + SUGGEST_CONTAINER_MEMORY_LIMIT, SUGGEST_DOCKER_INSTALL, SUGGEST_DOCKER_START, } from "../../command-internal/docker-suggest.ts"; @@ -12,8 +13,9 @@ import { /** * The edge runtime container exited with a real crash signal or other non-zero code while * streaming logs. Supervisor-initiated shutdowns (SIGHUP/SIGINT/SIGQUIT/SIGTERM) end the - * `functions serve` session successfully instead of reaching this error; 137 (SIGKILL) is - * retried by `streamContainerLogs` rather than failing. + * `functions serve` session successfully instead of reaching this error. `oomKilled` means the + * container hit its memory limit; a `137` without it was killed from outside the CLI, which we + * cannot attribute to either side. */ export class EdgeRuntimeContainerCrashedError extends Data.TaggedError( "EdgeRuntimeContainerCrashedError", @@ -21,10 +23,22 @@ export class EdgeRuntimeContainerCrashedError extends Data.TaggedError( readonly message: string; readonly containerId: string; readonly exitCode: number; + readonly oomKilled: boolean; }> { get [ErrorActionabilityId](): CliErrorActionabilityDeclaration { + if (this.oomKilled) { + return { ...actionability.resourceLimit, fingerprint_suffix: "out_of_memory" }; + } + if (this.exitCode === 137) { + return { ...actionability.unknown, fingerprint_suffix: "container_killed" }; + } return actionability.runtimeCrash; } + + /** Only an out-of-memory kill has a known remediation; other kills don't. */ + get suggestion(): string | undefined { + return this.oomKilled ? SUGGEST_CONTAINER_MEMORY_LIMIT : undefined; + } } /** diff --git a/apps/cli/src/shared/functions/serve.errors.unit.test.ts b/apps/cli/src/shared/functions/serve.errors.unit.test.ts index 2fd3920077..d8141a7378 100644 --- a/apps/cli/src/shared/functions/serve.errors.unit.test.ts +++ b/apps/cli/src/shared/functions/serve.errors.unit.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { + SUGGEST_CONTAINER_MEMORY_LIMIT, SUGGEST_DOCKER_INSTALL, SUGGEST_DOCKER_START, } from "../../command-internal/docker-suggest.ts"; @@ -18,6 +19,7 @@ describe("EdgeRuntimeContainerCrashedError actionability", () => { message: "error running container: exit 1", containerId: "abc123", exitCode: 1, + oomKilled: false, }); const result = classifyCliErrorActionability(error); @@ -30,6 +32,7 @@ describe("EdgeRuntimeContainerCrashedError actionability", () => { message: "error running container abc123: exit 143", containerId: "abc123", exitCode: 143, + oomKilled: false, }); const result = classifyCliErrorActionability(error); @@ -43,6 +46,7 @@ describe("EdgeRuntimeContainerCrashedError actionability", () => { message: "error running container abc123: exit 130", containerId: "abc123", exitCode: 130, + oomKilled: false, }); const result = classifyCliErrorActionability(error); @@ -55,12 +59,56 @@ describe("EdgeRuntimeContainerCrashedError actionability", () => { message: "error running container: exit 139", containerId: "abc123", exitCode: 139, + oomKilled: false, }); const result = classifyCliErrorActionability(error); expect(result.error_kind).toBe(actionability.runtimeCrash.error_kind); expect(result.error_category).toBe(actionability.runtimeCrash.error_category); }); + + it("classifies an out-of-memory kill (137, OOMKilled) as user-actionable, with a suggestion", () => { + const error = new EdgeRuntimeContainerCrashedError({ + message: "error running container abc123: exit 137", + containerId: "abc123", + exitCode: 137, + oomKilled: true, + }); + + expect(error.suggestion).toBe(SUGGEST_CONTAINER_MEMORY_LIMIT); + const result = classifyCliErrorActionability(error); + expect(result.error_kind).toBe(actionability.resourceLimit.error_kind); + expect(result.error_category).toBe(actionability.resourceLimit.error_category); + expect(result.error_fingerprint).toBe("tag:EdgeRuntimeContainerCrashedError:out_of_memory"); + }); + + it("classifies a non-OOM kill (137, not OOMKilled) as unknown, not our bug", () => { + const error = new EdgeRuntimeContainerCrashedError({ + message: "error running container abc123: exit 137", + containerId: "abc123", + exitCode: 137, + oomKilled: false, + }); + + expect(error.suggestion).toBeUndefined(); + const result = classifyCliErrorActionability(error); + expect(result.error_kind).toBe(actionability.unknown.error_kind); + expect(result.error_category).toBe(actionability.unknown.error_category); + expect(result.error_fingerprint).toBe("tag:EdgeRuntimeContainerCrashedError:container_killed"); + }); + + it("classifies OOMKilled as out-of-memory even with a non-137 exit code", () => { + const error = new EdgeRuntimeContainerCrashedError({ + message: "error running container abc123: exit 1", + containerId: "abc123", + exitCode: 1, + oomKilled: true, + }); + + const result = classifyCliErrorActionability(error); + expect(result.error_kind).toBe(actionability.resourceLimit.error_kind); + expect(result.error_category).toBe(actionability.resourceLimit.error_category); + }); }); describe("DockerLogsStreamError suggestion", () => { diff --git a/apps/cli/src/shared/functions/serve.ts b/apps/cli/src/shared/functions/serve.ts index eef059c0c1..9d1071de42 100644 --- a/apps/cli/src/shared/functions/serve.ts +++ b/apps/cli/src/shared/functions/serve.ts @@ -134,7 +134,8 @@ const dockerLogDiagnosticTailLength = 4_096; // signal and a child spawn/stream failure can land microseconds apart — this is their tie-break. const shutdownSignalGracePeriod = Duration.millis(50); // Exit codes a supervisor uses to tear a container down (`supabase stop`, CI cancellation), -// not a self-raised crash signal; 137 is excluded because it gets its own OOM-kill retry. +// not a self-raised crash signal; 137 is excluded because it needs `OOMKilled` to tell a +// memory-limit kill from a kill the CLI cannot attribute. const externalTerminationExitCodes = new Set([ 129, // SIGHUP 130, // SIGINT @@ -1423,15 +1424,12 @@ const streamContainerLogs = Effect.fnUntraced(function* ( if (state.exitCode === 0) { return { _tag: "containerExited" } satisfies ContainerLogsEndReason; } - if (state.exitCode === 137) { - yield* reattach; - continue; - } return yield* Effect.fail( new EdgeRuntimeContainerCrashedError({ message: `error running container ${containerId}: exit ${state.exitCode}`, containerId, exitCode: state.exitCode, + oomKilled: state.oomKilled, }), ); }