From c429678d358afbfc7e963de83bd1fc94395a2573 Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Sat, 14 Nov 2026 19:25:42 +0400 Subject: [PATCH] fix(deploy): wait for the new build before reporting the rollout Right after the pool PUT, Rivet Cloud can still return the pool status from before it, with the new build_id. Deploy read that old status as the result: a false success for a new build, and a false failure for a rollback from a degraded pool. A status now counts only after the pool was seen rolling, or when it differs from the status read before the PUT. The rest of the wait follows rivet deploy: nodes updated and ready, and degraded or down fails only when action_needed. A redeploy that changes nothing takes the current status. The rollback hint names the previous build only if it was ready. --- src/commands/deploy.ts | 38 +++++++++++-- src/commands/rivet-cloud-api.ts | 4 ++ tests/cli/orange-deploy.test.ts | 87 +++++++++++++++++++++++++++++ tests/harness/rivet-cloud-double.ts | 52 ++++++++++++----- 4 files changed, 162 insertions(+), 19 deletions(-) diff --git a/src/commands/deploy.ts b/src/commands/deploy.ts index 2564beb..44a95f8 100644 --- a/src/commands/deploy.ts +++ b/src/commands/deploy.ts @@ -278,8 +278,10 @@ async function executeDeployCommand(args: readonly string[], io: DeployCommandIo } // A secret's new value rolls every pool that names it, as `rivet cluster secret set` does, so // this waits until the build is ready and the pool rolls to it next. + let secretChanged = false; for (const [secret, value] of change?.uploads ?? []) { const result = await cloud.rolloutSecret(secret, value); + secretChanged ||= result.changed; io.log(`${result.changed ? "Rolled out" : "Unchanged:"} secret ${secret} (version ${result.latest_version})`); } if (sendEnvironment && next.unpinned.length > 0) { @@ -295,8 +297,17 @@ async function executeDeployCommand(args: readonly string[], io: DeployCommandIo ...(target.machineType === undefined ? {} : { machine_type: target.machineType }), ...(target.nodeCount === undefined ? {} : { node_count: target.nodeCount }), }); - const pool = await waitForRollout(cloud, theme, region.region, target.pool, () => - rollbackHint(cluster, region.region, target.pool, previous?.build_id ?? undefined, buildId), + // Only a build the pool ran as `ready` is safe to roll back to. + const lastReadyBuild = previous?.status.state === "ready" ? (previous.build_id ?? undefined) : undefined; + // With nothing to roll, the pool's status from before the PUT already describes this build. + const unchanged = + previous?.build_id === buildId && + !sendEnvironment && + !secretChanged && + target.machineType === undefined && + target.nodeCount === undefined; + const pool = await waitForRollout(cloud, theme, region.region, target.pool, buildId, unchanged ? undefined : previous, () => + rollbackHint(cluster, region.region, target.pool, lastReadyBuild, buildId), ); io.log(`${theme.fg("green", "✓")} Deployed ${reference} to pool ${target.pool} in ${region.region}`); io.log(` ${"Build".padEnd(8)}${buildId}`); @@ -440,22 +451,41 @@ function buildNotReady(build: RivetBuild, cluster: string | undefined): Error { return new ExpectedFailure(`Build ${build.build_id}${where} is ${build.status}, and only a ready build can be rolled out. Wait until it is ready, then deploy it again.`); } +// A read that still returns the status from before the PUT matches it exactly. +function sameStatus(a: RivetPool["status"], b: RivetPool["status"]): boolean { + return a.state === b.state && a.state_since === b.state_since && JSON.stringify(a.replicas) === JSON.stringify(b.replicas); +} + +// Waits as `rivet deploy` does (engine/packages/cli/src/commands/deploy/cluster.rs). Rivet Cloud +// can still return the status from before the PUT, so a status counts only once the pool was seen +// rolling or its status changed. Only Rivet Cloud's own values are compared, never the local clock. async function waitForRollout( cloud: RivetCloudClient, theme: Theme, region: string, poolName: string, + buildId: string, + // The pool before the PUT, or undefined when its status cannot be stale. + before: RivetPool | undefined, hint: () => string, ): Promise { const deadline = Date.now() + rolloutTimeoutMs; const spinner = startSpinner(process.stderr, theme, `Waiting for pool ${poolName} to roll out…`); + let rolling = false; try { for (;;) { const pool = await cloud.getPool(region, poolName); if (pool === undefined) throw new Error(`Pool ${poolName} in region ${region} disappeared during the rollout.`); + if (pool.build_id !== buildId) { + throw new ExpectedFailure(`Pool ${poolName} in region ${region} moved to build ${pool.build_id ?? "none"} during the rollout, so this deploy stopped.`); + } const { replicas, state } = pool.status; - if (state === "ready" && replicas !== null && replicas.updated === replicas.desired) return pool; - if (state === "degraded" || state === "down") { + if (state !== "ready" && state !== "degraded" && state !== "down") rolling = true; + const current = before === undefined || rolling || !sameStatus(pool.status, before.status); + const rolledOut = state === "ready" && replicas !== null && replicas.updated >= replicas.desired && replicas.ready >= replicas.desired; + if (current && rolledOut) return pool; + // As in `rivet deploy`, a pool that is degraded or down only fails the deploy when the user must act. + if (current && (state === "degraded" || state === "down") && pool.status.action_needed !== false) { const fault = pool.status.fault === null ? "" : ` (${pool.status.fault} fault)`; // A Worker that crashes on start is the user's to fix, not a bug in Orange. throw new (pool.status.fault === "user" ? ExpectedFailure : Error)( diff --git a/src/commands/rivet-cloud-api.ts b/src/commands/rivet-cloud-api.ts index 80dd385..c96b6ec 100644 --- a/src/commands/rivet-cloud-api.ts +++ b/src/commands/rivet-cloud-api.ts @@ -52,10 +52,14 @@ const poolSchema = z.object({ env: z.record(z.string(), envValueSchema).optional(), endpoints: z.object({ public: z.string().nullable() }), status: z.object({ + // The user must act on `message`, such as fixing the deployed build. + action_needed: z.boolean().optional(), fault: z.string().nullable(), message: z.string().nullable(), replicas: z.object({ desired: z.number(), ready: z.number(), updated: z.number() }).nullable(), state: z.string(), + // When the pool entered `state`, in milliseconds since the epoch. Null while Rivet Cloud has not observed it. + state_since: z.number().nullable().optional(), }), }); export type RivetPool = z.infer; diff --git a/tests/cli/orange-deploy.test.ts b/tests/cli/orange-deploy.test.ts index 93a19c8..5880c9d 100644 --- a/tests/cli/orange-deploy.test.ts +++ b/tests/cli/orange-deploy.test.ts @@ -215,6 +215,93 @@ test("a pool that goes down during the rollout fails with its reason and the rol ); }); +const crashLoop = { fault: "user", message: "Build `bld_broken` failed: App keeps exiting.", state: "degraded" } as const; + +// What Rivet Cloud reports for the pool when the deploy is over, as any API client reads it. +async function readPool(cloud: RivetCloudDouble, pool: string): Promise { + const response = await fetch(`${cloud.url}/v2/clusters/${cluster}/regions/iad/runner-pools/${pool}`, { + headers: { authorization: `Bearer ${token}` }, + }); + const { build_id, status } = (await response.json()) as { build_id: string; status: { replicas: unknown; state: string } }; + return { build_id, replicas: status.replicas, state: status.state }; +} + +test("a deploy prints success only after the new build rolled out, even when the first read still shows the old ready pool", async () => { + await withDeploy( + { builds: { bld_new: "ready", bld_old: "ready" }, pools: { "iad/default": existingPool }, staleReads: 1 }, + async (cloud, project, env) => { + const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--build", "bld_new"], env); + assert.equal(code, 0, output); + assert.match(output, /✓ Deployed .* to pool default in iad/); + assert.deepEqual(await readPool(cloud, "default"), { + build_id: "bld_new", + replicas: { desired: 2, ready: 2, updated: 2 }, + state: "ready", + }); + }, + ); +}); + +test("a deploy whose Worker keeps crashing fails, even when the first read still shows the old ready pool", async () => { + await withDeploy( + { builds: { bld_broken: "ready", bld_old: "ready" }, poolOutcome: crashLoop, pools: { "iad/default": existingPool }, staleReads: 1 }, + async (cloud, project, env) => { + const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--build", "bld_broken"], env); + assert.equal(code, 1, output); + assert.doesNotMatch(output, /Deployed/); + assert.match(output, /Pool default in region iad is degraded \(user fault\): Build `bld_broken` failed: App keeps exiting\./); + assert.match(output, /Roll back with: orange deploy --cluster acme --region iad --pool default --build bld_old/); + }, + ); +}); + +test("a rollback from a degraded pool succeeds, even when the first read still shows the old degraded status", async () => { + await withDeploy( + { + builds: { bld_broken: "ready", bld_old: "ready" }, + pools: { "iad/default": { ...existingPool, build_id: "bld_broken" } }, + poolStatus: crashLoop, + staleReads: 1, + }, + async (cloud, project, env) => { + const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--build", "bld_old"], env); + assert.equal(code, 0, output); + assert.doesNotMatch(output, /degraded|Roll back with/); + assert.match(output, /✓ Deployed .* to pool default in iad/); + }, + ); +}); + +test("a failed rollout on a degraded pool never suggests rolling back to the broken build", async () => { + await withDeploy( + { + builds: { bld_broken: "ready", bld_new: "ready" }, + poolOutcome: { fault: "user", message: "Runners exit right after they start.", state: "down" }, + pools: { "iad/default": { ...existingPool, build_id: "bld_broken" } }, + poolStatus: crashLoop, + }, + async (cloud, project, env) => { + const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--build", "bld_new"], env); + assert.equal(code, 1, output); + assert.match(output, /Pool default in region iad is down \(user fault\): Runners exit right after they start\./); + assert.doesNotMatch(output, /Roll back with/); + }, + ); +}); + +test("redeploying the build a ready pool already runs succeeds at once, even when Rivet Cloud never reports a new status", async () => { + await withDeploy( + { builds: { bld_old: "ready" }, pools: { "iad/default": existingPool }, staleReads: 1_000 }, + async (cloud, project, env) => { + const started = Date.now(); + const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--build", "bld_old"], env); + assert.equal(code, 0, output); + assert.match(output, /✓ Deployed .* to pool default in iad/); + assert.ok(Date.now() - started < 30_000, `the deploy waited ${Date.now() - started} ms`); + }, + ); +}); + test("--dry-run writes a credential-free runner image and calls no API", async () => { await withDeploy({}, async (cloud, project) => { await writeFile(join(project.directory, ".dev.vars"), "API_KEY=dev-secret-value\n"); diff --git a/tests/harness/rivet-cloud-double.ts b/tests/harness/rivet-cloud-double.ts index 5c4ef55..60265be 100644 --- a/tests/harness/rivet-cloud-double.ts +++ b/tests/harness/rivet-cloud-double.ts @@ -19,6 +19,8 @@ export interface CloudPool { node_count: number; } +export type PoolOutcome = "ready" | { fault: "platform" | "user"; message: string; state: "degraded" | "down" }; + export interface RivetCloudDoubleOptions { cluster: string; // Builds that exist before the deploy, by ID, with their status. @@ -28,7 +30,11 @@ export interface RivetCloudDoubleOptions { // Pools that exist before the deploy, keyed `/`. pools?: Record; // What a rolled-out pool becomes after one `updating` read. Default `ready`. - poolOutcome?: "ready" | { fault: "platform" | "user"; message: string; state: "degraded" | "down" }; + poolOutcome?: PoolOutcome; + // The status of the pools that exist before the deploy. Default `ready`. + poolStatus?: PoolOutcome; + // Reads right after a PUT that still return the status from before it, as Rivet Cloud can. Default 0. + staleReads?: number; regions: string[]; token: string; // `token` is a cluster token, which Rivet Cloud lets update a pool but not create one. @@ -81,6 +87,7 @@ interface BuildRecord { interface PoolState { reads: number; rolling: boolean; + since: number; } export async function startRivetCloudDouble(options: RivetCloudDoubleOptions): Promise { @@ -142,29 +149,44 @@ export async function startRivetCloudDouble(options: RivetCloudDoubleOptions): P upload_expires_at: new Date(Date.now() + 3_600_000).toISOString(), }); const regionPublicUrl = (region: string): string => `https://${region}.${options.cluster}.rivet.test`; - // A rolled-out pool reads as `updating` once, then as its outcome. - const poolView = (key: string, pool: CloudPool, read: boolean): Record => { - const [region = "", name = ""] = key.split("/"); - const state = poolStates.get(key); - const desired = pool.node_count; - let status: Record = { + const outcomeStatus = (outcome: PoolOutcome, desired: number, since: number): Record => { + const ready = { action_needed: false, fault: null, message: null, replicas: { desired, ready: desired, updated: desired }, state: "ready", - state_since: 0, + state_since: since, waiting_on: null, }; + if (outcome === "ready") return ready; + return { + ...ready, + action_needed: true, + fault: outcome.fault, + message: outcome.message, + replicas: { desired, ready: 0, updated: 0 }, + state: outcome.state, + }; + }; + // A rolled-out pool reads as its old status `staleReads` times, as `updating` once, then as its outcome. + const poolView = (key: string, pool: CloudPool, read: boolean): Record => { + const [region = "", name = ""] = key.split("/"); + const state = poolStates.get(key); + const desired = pool.node_count; + const before = outcomeStatus(options.poolStatus ?? "ready", desired, 0); + let status = state === undefined ? before : outcomeStatus("ready", desired, state.since); if (state?.rolling === true) { if (read) state.reads += 1; - const outcome = options.poolOutcome ?? "ready"; - if (state.reads <= 1) { - status = { ...status, replicas: { desired, ready: desired, updated: 0 }, state: "updating", waiting_on: "runners" }; - } else if (outcome !== "ready") { - status = { ...status, action_needed: true, fault: outcome.fault, message: outcome.message, replicas: { desired, ready: 0, updated: 0 }, state: outcome.state }; + const stale = options.staleReads ?? 0; + if (state.reads <= stale) { + status = before; + } else if (state.reads <= stale + 1) { + status = { ...status, replicas: { desired, ready: desired, updated: 0 }, state: "updating", state_since: state.since, waiting_on: "runners" }; } else { - state.rolling = false; + if (state.reads === stale + 2) state.since = Date.now(); + status = outcomeStatus(options.poolOutcome ?? "ready", desired, state.since); + if (options.poolOutcome === undefined || options.poolOutcome === "ready") state.rolling = false; } } return { @@ -386,7 +408,7 @@ export async function startRivetCloudDouble(options: RivetCloudDoubleOptions): P node_count: (body.node_count as number | undefined) ?? existing?.node_count ?? 1, }; pools.set(key, pool); - poolStates.set(key, { reads: 0, rolling: true }); + poolStates.set(key, { reads: 0, rolling: true, since: Date.now() }); send(response, 200, poolView(key, pool, false)); return; }