test(cli): reconcile live test resource lifecycles

apps/cli/scripts/sweep-live-projects.integration.test.ts

@@ -0,0 +1,162 @@
+import { afterEach, describe, expect, test } from "vitest";
+import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+const script = path.resolve(import.meta.dirname, "sweep-live-projects.sh");
+const directories: string[] = [];
+
+afterEach(async () => {
+  await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true })));
+});
+
+type Scenario = {
+  lists: Array<unknown> | ((deletes: ReadonlyArray<string>) => unknown);
+  deletes: Record<string, { status: number; body?: unknown }>;
+  deleteStatuses?: Record<string, Array<number>>;
+};
+
+async function runSweep(scenario: Scenario) {
+  const directory = await mkdtemp(path.join(tmpdir(), "sweep-live-projects-"));
+  directories.push(directory);
+  const fakeSleep = path.join(directory, "sleep");
+  await writeFile(fakeSleep, "#!/bin/sh\nexit 0\n");
+  await chmod(fakeSleep, 0o755);
+
+  let listIndex = 0;
+  const deletes: string[] = [];
+  const server = Bun.serve({
+    port: 0,
+    fetch(request) {
+      const url = new URL(request.url);
+      const ref = url.pathname.split("/").pop() ?? "";
+      if (request.method === "GET" && url.pathname === "/v1/projects") {
+        const value =
+          typeof scenario.lists === "function"
+            ? scenario.lists(deletes)
+            : scenario.lists[Math.min(listIndex++, scenario.lists.length - 1)];
+        return typeof value === "number"
+          ? new Response("temporary failure", { status: value })
+          : Response.json(value);
+      }
+      if (request.method === "DELETE" && url.pathname.startsWith("/v1/projects/")) {
+        deletes.push(ref);
+        const statuses = scenario.deleteStatuses?.[ref];
+        const result: { status: number; body?: unknown } =
+          statuses === undefined
+            ? (scenario.deletes[ref] ?? { status: 404 })
+            : {
+                status:
+                  statuses[
+                    Math.min(
+                      deletes.filter((value) => value === ref).length - 1,
+                      statuses.length - 1,
+                    )
+                  ] ?? 500,
+              };
+        return new Response(result.body === undefined ? null : JSON.stringify(result.body), {
+          status: result.status,
+          headers: { "x-request-id": `delete-${ref}` },
+        });
+      }
+      return new Response("not found", { status: 404 });
+    },
+  });
+  try {
+    const child = Bun.spawn(["bash", script, "e2e-"], {
+      env: {
+        ...globalThis.process.env,
+        PATH: `${directory}:${globalThis.process.env.PATH}`,
+        SUPABASE_ACCESS_TOKEN: "test-token",
+        SUPABASE_LIVE_API_URL: `http://127.0.0.1:${server.port}`,
+      },
+      stdout: "pipe",
+      stderr: "pipe",
+    });
+    const timeout = setTimeout(() => child.kill(), 10_000);
+    const [exitCode, stdout, stderr] = await Promise.all([
+      child.exited,
+      new Response(child.stdout).text(),
+      new Response(child.stderr).text(),
+    ]);
+    clearTimeout(timeout);
+    return { exitCode, stdout, stderr, deletes };
+  } finally {
+    await server.stop(true);
+  }
+}
+
+const active = (ref: string, name = `e2e-${ref}`) => ({ ref, name, status: "ACTIVE" });
+
+describe.skipIf(process.platform === "win32")("sweep-live-projects.sh", { timeout: 40_000 }, () => {
+  test("accepts refused deletion when a fresh authenticated list shows absence", async () => {
+    const result = await runSweep({
+      lists: [[active("gone")], []],
+      deletes: { gone: { status: 403, body: { message: "already removed" } } },
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.deletes).toEqual(["gone"]);
+    expect(`${result.stdout}\n${result.stderr}`).not.toContain("test-token");
+  });
+
+  test("accepts a terminal status and converges after a stale list", async () => {
+    const result = await runSweep({
+      lists: [[active("stale")], [active("stale")], [{ ...active("stale"), status: "REMOVED" }]],
+      deletes: { stale: { status: 202 } },
+    });
+    expect(result.exitCode).toBe(0);
+  });
+
+  test("retries a transient read failure but rejects malformed evidence", async () => {
+    const transient = await runSweep({
+      lists: [503, [active("retry")], []],
+      deletes: { retry: { status: 202 } },
+    });
+    expect(transient.exitCode).toBe(0);
+
+    const malformed = await runSweep({ lists: [{ projects: [] }], deletes: {} });
+    expect(malformed.exitCode).not.toBe(0);
+    expect(malformed.deletes).toEqual([]);
+  });
+
+  test("fails terminal listing errors without retrying or deleting", async () => {
+    const forbidden = await runSweep({ lists: [403, []], deletes: {} });
+    expect(forbidden.exitCode).not.toBe(0);
+    expect(forbidden.deletes).toEqual([]);
+
+    const unavailable = await runSweep({ lists: [503, 503, 503], deletes: {} });
+    expect(unavailable.exitCode).not.toBe(0);
+    expect(unavailable.deletes).toEqual([]);
+
+    const malformedReconciliation = await runSweep({
+      lists: [[active("bad-evidence")], { projects: [] }],
+      deletes: { "bad-evidence": { status: 403 } },
+    });
+    expect(malformedReconciliation.exitCode).not.toBe(0);
+  });
+
+  test("attempts every owned project and fails if one remains active", async () => {
+    const result = await runSweep({
+      lists: [
+        [active("stuck"), active("other"), { ref: "foreign", name: "unrelated", status: "ACTIVE" }],
+        [active("stuck"), { ref: "foreign", name: "unrelated", status: "ACTIVE" }],
+      ],
+      deletes: { stuck: { status: 403 }, other: { status: 204 } },
+    });
+    expect(result.exitCode).not.toBe(0);
+    expect(result.deletes).toEqual(["stuck", "other"]);
+  });
+
+  test("retries a transient deletion after fresh evidence still shows the project", async () => {
+    const result = await runSweep({
+      lists: (deletes) => (deletes.length >= 2 ? [] : [active("flaky")]),
+      deletes: {},
+      deleteStatuses: { flaky: [503, 204] },
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.deletes).toEqual(["flaky", "flaky"]);
+    expect(result.stderr).toContain(
+      "delete retry completed for project flaky (HTTP 204 request delete-flaky)",
+    );
+  });
+});

apps/cli/scripts/sweep-live-projects.sh

@@ -1,67 +1,112 @@
 #!/usr/bin/env bash
-# Delete every staging project whose name starts with the given prefix (the live
-# e2e job's per-run prefix). Runs as the live e2e job's always() cleanup step,
-# which propagates the exit code.
-#
-# Reads SUPABASE_ACCESS_TOKEN + SUPABASE_LIVE_API_URL from the environment. Exits
-# non-zero if any DELETE failed and the project still reads as live; a failed
-# *listing* also exits non-zero.
+# Delete every staging project whose name starts with the live e2e prefix.
 set -eo pipefail
 
 PREFIX="${1:?usage: sweep-live-projects.sh PREFIX}"
 : "${SUPABASE_ACCESS_TOKEN:?SUPABASE_ACCESS_TOKEN required}"
 : "${SUPABASE_LIVE_API_URL:?SUPABASE_LIVE_API_URL required}"
 
-# Retry into a file, not a pipe: curl rewinds a file between attempts, but a
-# pipe keeps an already-flushed body and the retry would duplicate the listing.
 listing=$(mktemp)
-project=$(mktemp)
-trap 'rm -f "$listing" "$project"' EXIT
-curl -fsS --retry 3 --retry-connrefused --max-time 60 --retry-max-time 120 \
-  -H "Authorization: Bearer ${SUPABASE_ACCESS_TOKEN}" \
-  -o "$listing" "${SUPABASE_LIVE_API_URL}/v1/projects"
-# Capture the list in a var (not a pipe-to-while subshell) so a failed delete is
-# recorded in $failed; a failed listing aborts above via errexit.
-projects=$(jq -r --arg p "$PREFIX" '.[] | select(.name|startswith($p)) | "\(.ref // .id) \(.status)"' "$listing")
+headers=$(mktemp)
+response=$(mktemp)
+trap 'rm -f "$listing" "$headers" "$response"' EXIT
 
-# The suite's teardown can delete a project after the listing was taken, so a
-# refused DELETE only counts once the project still reads as live afterwards
-# (12 bounded reads, 5s apart); a removed project is refused or GOING_DOWN.
-gone() {
-  local attempt code status
+api_request() {
+  local method=$1 url=$2 output=$3
+  : > "$headers"
+  : > "$output"
+  API_CODE=$(curl -sS --max-time 15 -X "$method" -D "$headers" -o "$output" \
+    -H "Authorization: Bearer ${SUPABASE_ACCESS_TOKEN}" -w '%{http_code}' "$url") || API_CODE=000
+  API_REQUEST_ID=$(awk 'tolower($0) ~ /^x-request-id:/ { sub(/^[^:]*:[[:space:]]*/, ""); gsub(/[[:space:]]+$/, ""); print; exit }' "$headers" | tr -cd '[:alnum:]_.:/-' | cut -c1-120)
+}
+
+fetch_listing_once() {
+  api_request GET "${SUPABASE_LIVE_API_URL}/v1/projects" "$listing"
+  if [ "$API_CODE" != 200 ]; then
+    echo "project listing failed (HTTP $API_CODE${API_REQUEST_ID:+ request $API_REQUEST_ID})" >&2
+    case "$API_CODE" in 000|408|429|5??) return 1 ;; esac
+    return 2
+  fi
+  if ! jq -e 'type == "array" and all(.[]; (.name | type == "string") and ((.ref // .id) | type == "string") and ((.status // "") | type == "string"))' "$listing" >/dev/null; then
+    echo "project listing was malformed (HTTP 200${API_REQUEST_ID:+ request $API_REQUEST_ID})" >&2
+    return 2
+  fi
+}
+
+initial_attempt=1
+while true; do
+  if fetch_listing_once; then
+    initial_result=0
+    break
+  else
+    initial_result=$?
+  fi
+  [ "$initial_result" -eq 2 ] && break
+  [ "$initial_attempt" -lt 3 ] || break
+  initial_attempt=$((initial_attempt + 1))
+  sleep 5
+done
+if [ "$initial_result" -ne 0 ]; then
+  echo "::error::unable to obtain a complete authenticated project listing" >&2
+  exit 1
+fi
+
+projects=$(jq -r --arg p "$PREFIX" '.[] | select(.name | startswith($p)) | "\(.ref // .id) \(.status // "")"' "$listing")
+
+# Reconcile through fresh authenticated list responses. A DELETE 403/400 or a
+# single-project 404 is not evidence that a project is gone.
+reconcile() {
+  local ref=$1 attempt
   for attempt in $(seq 12); do
-    # curl leaves the output file untouched when no body arrives.
-    : > "$project"
-    code=$(curl -sS --max-time 10 -o "$project" -w '%{http_code}' \
-      -H "Authorization: Bearer ${SUPABASE_ACCESS_TOKEN}" \
-      "${SUPABASE_LIVE_API_URL}/v1/projects/$1") || code=000
-    status=$(jq -r '.status // empty' "$project" 2>/dev/null || true)
-    case "$code:$status" in
-      403:* | 404:* | 200:GOING_DOWN | 200:REMOVED)
-        echo "project $1 already gone ($code${status:+ $status})"
+    if fetch_listing_once; then
+      if jq -e --arg ref "$ref" 'any(.[]; ((.ref // .id) == $ref) and ((.status // "") != "GOING_DOWN") and ((.status // "") != "REMOVED"))' "$listing" >/dev/null; then
+        echo "project $ref still active (list HTTP $API_CODE${API_REQUEST_ID:+ request $API_REQUEST_ID})" >&2
+        if [ "$delete_attempt" -lt 3 ]; then
+          case "$delete_code" in
+            000|408|425|429|5??)
+              delete_attempt=$((delete_attempt + 1))
+              echo "retrying transient delete for project $ref (attempt $delete_attempt)" >&2
+              sleep 5
+              api_request DELETE "${SUPABASE_LIVE_API_URL}/v1/projects/${ref}" "$response"
+              delete_code=$API_CODE
+              echo "delete retry completed for project $ref (HTTP $delete_code${API_REQUEST_ID:+ request $API_REQUEST_ID})" >&2
+              ;;
+          esac
+        fi
+      else
+        echo "project $ref reconciled as absent or terminal (list HTTP $API_CODE${API_REQUEST_ID:+ request $API_REQUEST_ID})"
         return 0
-        ;;
-    esac
+      fi
+    elif [ "$?" -eq 2 ]; then
+      echo "project $ref reconciliation evidence was unavailable or malformed" >&2
+      return 1
+    else
+      echo "project $ref reconciliation read failed (HTTP $API_CODE${API_REQUEST_ID:+ request $API_REQUEST_ID})" >&2
+    fi
     [ "$attempt" -lt 12 ] && sleep 5
   done
-  echo "project $1 still reads $code${status:+ $status}"
   return 1
 }
 
 failed=0
 while read -r ref status; do
   [ -n "$ref" ] || continue
-  # Already deleted by the suite; a second DELETE is refused.
   case "$status" in
     GOING_DOWN | REMOVED)
       echo "skipping project $ref ($status)"
       continue
       ;;
   esac
-  echo "deleting leftover project $ref"
-  if ! curl -fsS -X DELETE -H "Authorization: Bearer ${SUPABASE_ACCESS_TOKEN}" \
-    "${SUPABASE_LIVE_API_URL}/v1/projects/${ref}" >/dev/null && ! gone "$ref"; then
-    echo "::error::failed to delete leftover project $ref"
+  delete_attempt=1
+  echo "deleting leftover project $ref (attempt $delete_attempt)"
+  api_request DELETE "${SUPABASE_LIVE_API_URL}/v1/projects/${ref}" "$response"
+  delete_code=$API_CODE
+  case "$API_CODE" in
+    2??) echo "delete accepted for project $ref (HTTP $API_CODE${API_REQUEST_ID:+ request $API_REQUEST_ID})" ;;
+    *) echo "delete failed for project $ref (HTTP $API_CODE${API_REQUEST_ID:+ request $API_REQUEST_ID})" >&2 ;;
+  esac
+  if ! reconcile "$ref"; then
+    echo "::error::project $ref remains active or cleanup evidence was unavailable" >&2
     failed=1
   fi
 done <<< "$projects"

apps/cli/src/commands/branches/create/create.live.test.ts

@@ -1,28 +1,55 @@
 import { randomUUID } from "node:crypto";
 
-import { Cause, Effect, Exit } from "effect";
+import { Cause, Effect, Exit, Schema } from "effect";
 import { expect } from "vitest";
 
-import { test, throwWithCleanup } from "../../../../tests/helpers/live.ts";
-import { awaitBranch, removeBranch } from "../../../../tests/helpers/branches-live.ts";
+import { throwWithCleanup, test } from "../../../../tests/helpers/live.ts";
+import {
+  awaitLiveBranchEffect,
+  awaitLiveBranchRemovedEffect,
+  removeLiveBranchByNameEffect,
+} from "../../../../tests/helpers/branches-live.ts";
 
-// Not wired to the test `signal`: cleanup runs through the plain-promise `cli` path, so an
-// interrupt would abandon an in-flight `branches delete` rather than stop it, leaving the
-// branch behind. Its own exit timeout bounds the wait instead.
-test("creates a preview branch", ({ cli, cliEffect, project }) =>
+const CreatedBranchRef = Schema.Struct({ project_ref: Schema.String });
+const CreatedBranch = Schema.Struct({ message: Schema.String, project_ref: Schema.String });
+
+test("creates a preview branch", ({ cliEffect, project }) =>
   Effect.runPromise(
     Effect.gen(function* () {
       const name = `cli-e2e-create-${randomUUID().slice(0, 8)}`;
+      let branchRef: string | undefined;
 
       const target = Effect.gen(function* () {
-        const result = yield* cliEffect(["branches", "create", name, "--project-ref", project.ref]);
+        const result = yield* cliEffect([
+          "branches",
+          "create",
+          name,
+          "--project-ref",
+          project.ref,
+          "--output-format",
+          "json",
+        ]);
         expect(result.exitCode, result.stderr).toBe(0);
-        expect(result.stdout).toContain("Created preview branch");
-        yield* awaitBranch(cli, project, name);
+        const refBody = yield* Schema.decodeEffect(Schema.fromJsonString(CreatedBranchRef))(
+          result.stdout,
+        );
+        branchRef = refBody.project_ref.length > 0 ? refBody.project_ref : undefined;
+        const body = yield* Schema.decodeEffect(Schema.fromJsonString(CreatedBranch))(
+          result.stdout,
+        );
+        expect(body).toMatchObject({
+          message: "Created preview branch",
+          project_ref: expect.any(String),
+        });
+        expect(branchRef).toBeDefined();
+        yield* awaitLiveBranchEffect(cliEffect, project, name);
       });
 
       const targetExit = yield* Effect.exit(target);
-      const cleanupExit = yield* Effect.exit(removeBranch(cli, project, name));
+      const cleanupExit =
+        branchRef === undefined
+          ? yield* Effect.exit(removeLiveBranchByNameEffect(cliEffect, project, name))
+          : yield* Effect.exit(awaitLiveBranchRemovedEffect(cliEffect, project, branchRef));
       return {
         targetError: Exit.isFailure(targetExit) ? Cause.squash(targetExit.cause) : undefined,
         cleanupErrors: Exit.isFailure(cleanupExit) ? [Cause.squash(cleanupExit.cause)] : [],