Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 34 additions & 4 deletions src/commands/deploy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -278,8 +278,10 @@ async function executeDeployCommand(args: readonly string[], io: DeployCommandIo
}
// A secret's new value rolls every pool that names it, as `rivet cluster secret set` does, so
// this waits until the build is ready and the pool rolls to it next.
let secretChanged = false;
for (const [secret, value] of change?.uploads ?? []) {
const result = await cloud.rolloutSecret(secret, value);
secretChanged ||= result.changed;
io.log(`${result.changed ? "Rolled out" : "Unchanged:"} secret ${secret} (version ${result.latest_version})`);
}
if (sendEnvironment && next.unpinned.length > 0) {
Expand All @@ -295,8 +297,17 @@ async function executeDeployCommand(args: readonly string[], io: DeployCommandIo
...(target.machineType === undefined ? {} : { machine_type: target.machineType }),
...(target.nodeCount === undefined ? {} : { node_count: target.nodeCount }),
});
const pool = await waitForRollout(cloud, theme, region.region, target.pool, () =>
rollbackHint(cluster, region.region, target.pool, previous?.build_id ?? undefined, buildId),
// Only a build the pool ran as `ready` is safe to roll back to.
const lastReadyBuild = previous?.status.state === "ready" ? (previous.build_id ?? undefined) : undefined;
// With nothing to roll, the pool's status from before the PUT already describes this build.
const unchanged =
previous?.build_id === buildId &&
!sendEnvironment &&
!secretChanged &&
target.machineType === undefined &&
target.nodeCount === undefined;
const pool = await waitForRollout(cloud, theme, region.region, target.pool, buildId, unchanged ? undefined : previous, () =>
rollbackHint(cluster, region.region, target.pool, lastReadyBuild, buildId),
);
io.log(`${theme.fg("green", "✓")} Deployed ${reference} to pool ${target.pool} in ${region.region}`);
io.log(` ${"Build".padEnd(8)}${buildId}`);
Expand Down Expand Up @@ -440,22 +451,41 @@ function buildNotReady(build: RivetBuild, cluster: string | undefined): Error {
return new ExpectedFailure(`Build ${build.build_id}${where} is ${build.status}, and only a ready build can be rolled out. Wait until it is ready, then deploy it again.`);
}

// A read that still returns the status from before the PUT matches it exactly.
function sameStatus(a: RivetPool["status"], b: RivetPool["status"]): boolean {
return a.state === b.state && a.state_since === b.state_since && JSON.stringify(a.replicas) === JSON.stringify(b.replicas);
}

// Waits as `rivet deploy` does (engine/packages/cli/src/commands/deploy/cluster.rs). Rivet Cloud
// can still return the status from before the PUT, so a status counts only once the pool was seen
// rolling or its status changed. Only Rivet Cloud's own values are compared, never the local clock.
async function waitForRollout(
cloud: RivetCloudClient,
theme: Theme,
region: string,
poolName: string,
buildId: string,
// The pool before the PUT, or undefined when its status cannot be stale.
before: RivetPool | undefined,
hint: () => string,
): Promise<RivetPool> {
const deadline = Date.now() + rolloutTimeoutMs;
const spinner = startSpinner(process.stderr, theme, `Waiting for pool ${poolName} to roll out…`);
let rolling = false;
try {
for (;;) {
const pool = await cloud.getPool(region, poolName);
if (pool === undefined) throw new Error(`Pool ${poolName} in region ${region} disappeared during the rollout.`);
if (pool.build_id !== buildId) {
throw new ExpectedFailure(`Pool ${poolName} in region ${region} moved to build ${pool.build_id ?? "none"} during the rollout, so this deploy stopped.`);
}
const { replicas, state } = pool.status;
if (state === "ready" && replicas !== null && replicas.updated === replicas.desired) return pool;
if (state === "degraded" || state === "down") {
if (state !== "ready" && state !== "degraded" && state !== "down") rolling = true;
const current = before === undefined || rolling || !sameStatus(pool.status, before.status);
const rolledOut = state === "ready" && replicas !== null && replicas.updated >= replicas.desired && replicas.ready >= replicas.desired;
if (current && rolledOut) return pool;
// As in `rivet deploy`, a pool that is degraded or down only fails the deploy when the user must act.
if (current && (state === "degraded" || state === "down") && pool.status.action_needed !== false) {
const fault = pool.status.fault === null ? "" : ` (${pool.status.fault} fault)`;
// A Worker that crashes on start is the user's to fix, not a bug in Orange.
throw new (pool.status.fault === "user" ? ExpectedFailure : Error)(
Expand Down
4 changes: 4 additions & 0 deletions src/commands/rivet-cloud-api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,14 @@ const poolSchema = z.object({
env: z.record(z.string(), envValueSchema).optional(),
endpoints: z.object({ public: z.string().nullable() }),
status: z.object({
// The user must act on `message`, such as fixing the deployed build.
action_needed: z.boolean().optional(),
fault: z.string().nullable(),
message: z.string().nullable(),
replicas: z.object({ desired: z.number(), ready: z.number(), updated: z.number() }).nullable(),
state: z.string(),
// When the pool entered `state`, in milliseconds since the epoch. Null while Rivet Cloud has not observed it.
state_since: z.number().nullable().optional(),
}),
});
export type RivetPool = z.infer<typeof poolSchema>;
Expand Down
87 changes: 87 additions & 0 deletions tests/cli/orange-deploy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,93 @@ test("a pool that goes down during the rollout fails with its reason and the rol
);
});

const crashLoop = { fault: "user", message: "Build `bld_broken` failed: App keeps exiting.", state: "degraded" } as const;

// What Rivet Cloud reports for the pool when the deploy is over, as any API client reads it.
async function readPool(cloud: RivetCloudDouble, pool: string): Promise<unknown> {
const response = await fetch(`${cloud.url}/v2/clusters/${cluster}/regions/iad/runner-pools/${pool}`, {
headers: { authorization: `Bearer ${token}` },
});
const { build_id, status } = (await response.json()) as { build_id: string; status: { replicas: unknown; state: string } };
return { build_id, replicas: status.replicas, state: status.state };
}

test("a deploy prints success only after the new build rolled out, even when the first read still shows the old ready pool", async () => {
await withDeploy(
{ builds: { bld_new: "ready", bld_old: "ready" }, pools: { "iad/default": existingPool }, staleReads: 1 },
async (cloud, project, env) => {
const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--build", "bld_new"], env);
assert.equal(code, 0, output);
assert.match(output, /✓ Deployed .* to pool default in iad/);
assert.deepEqual(await readPool(cloud, "default"), {
build_id: "bld_new",
replicas: { desired: 2, ready: 2, updated: 2 },
state: "ready",
});
},
);
});

test("a deploy whose Worker keeps crashing fails, even when the first read still shows the old ready pool", async () => {
await withDeploy(
{ builds: { bld_broken: "ready", bld_old: "ready" }, poolOutcome: crashLoop, pools: { "iad/default": existingPool }, staleReads: 1 },
async (cloud, project, env) => {
const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--build", "bld_broken"], env);
assert.equal(code, 1, output);
assert.doesNotMatch(output, /Deployed/);
assert.match(output, /Pool default in region iad is degraded \(user fault\): Build `bld_broken` failed: App keeps exiting\./);
assert.match(output, /Roll back with: orange deploy --cluster acme --region iad --pool default --build bld_old/);
},
);
});

test("a rollback from a degraded pool succeeds, even when the first read still shows the old degraded status", async () => {
await withDeploy(
{
builds: { bld_broken: "ready", bld_old: "ready" },
pools: { "iad/default": { ...existingPool, build_id: "bld_broken" } },
poolStatus: crashLoop,
staleReads: 1,
},
async (cloud, project, env) => {
const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--build", "bld_old"], env);
assert.equal(code, 0, output);
assert.doesNotMatch(output, /degraded|Roll back with/);
assert.match(output, /✓ Deployed .* to pool default in iad/);
},
);
});

test("a failed rollout on a degraded pool never suggests rolling back to the broken build", async () => {
await withDeploy(
{
builds: { bld_broken: "ready", bld_new: "ready" },
poolOutcome: { fault: "user", message: "Runners exit right after they start.", state: "down" },
pools: { "iad/default": { ...existingPool, build_id: "bld_broken" } },
poolStatus: crashLoop,
},
async (cloud, project, env) => {
const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--build", "bld_new"], env);
assert.equal(code, 1, output);
assert.match(output, /Pool default in region iad is down \(user fault\): Runners exit right after they start\./);
assert.doesNotMatch(output, /Roll back with/);
},
);
});

test("redeploying the build a ready pool already runs succeeds at once, even when Rivet Cloud never reports a new status", async () => {
await withDeploy(
{ builds: { bld_old: "ready" }, pools: { "iad/default": existingPool }, staleReads: 1_000 },
async (cloud, project, env) => {
const started = Date.now();
const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--build", "bld_old"], env);
assert.equal(code, 0, output);
assert.match(output, /✓ Deployed .* to pool default in iad/);
assert.ok(Date.now() - started < 30_000, `the deploy waited ${Date.now() - started} ms`);
},
);
});

test("--dry-run writes a credential-free runner image and calls no API", async () => {
await withDeploy({}, async (cloud, project) => {
await writeFile(join(project.directory, ".dev.vars"), "API_KEY=dev-secret-value\n");
Expand Down
52 changes: 37 additions & 15 deletions tests/harness/rivet-cloud-double.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ export interface CloudPool {
node_count: number;
}

export type PoolOutcome = "ready" | { fault: "platform" | "user"; message: string; state: "degraded" | "down" };

export interface RivetCloudDoubleOptions {
cluster: string;
// Builds that exist before the deploy, by ID, with their status.
Expand All @@ -28,7 +30,11 @@ export interface RivetCloudDoubleOptions {
// Pools that exist before the deploy, keyed `<region>/<pool>`.
pools?: Record<string, CloudPool>;
// What a rolled-out pool becomes after one `updating` read. Default `ready`.
poolOutcome?: "ready" | { fault: "platform" | "user"; message: string; state: "degraded" | "down" };
poolOutcome?: PoolOutcome;
// The status of the pools that exist before the deploy. Default `ready`.
poolStatus?: PoolOutcome;
// Reads right after a PUT that still return the status from before it, as Rivet Cloud can. Default 0.
staleReads?: number;
regions: string[];
token: string;
// `token` is a cluster token, which Rivet Cloud lets update a pool but not create one.
Expand Down Expand Up @@ -81,6 +87,7 @@ interface BuildRecord {
interface PoolState {
reads: number;
rolling: boolean;
since: number;
}

export async function startRivetCloudDouble(options: RivetCloudDoubleOptions): Promise<RivetCloudDouble> {
Expand Down Expand Up @@ -142,29 +149,44 @@ export async function startRivetCloudDouble(options: RivetCloudDoubleOptions): P
upload_expires_at: new Date(Date.now() + 3_600_000).toISOString(),
});
const regionPublicUrl = (region: string): string => `https://${region}.${options.cluster}.rivet.test`;
// A rolled-out pool reads as `updating` once, then as its outcome.
const poolView = (key: string, pool: CloudPool, read: boolean): Record<string, unknown> => {
const [region = "", name = ""] = key.split("/");
const state = poolStates.get(key);
const desired = pool.node_count;
let status: Record<string, unknown> = {
const outcomeStatus = (outcome: PoolOutcome, desired: number, since: number): Record<string, unknown> => {
const ready = {
action_needed: false,
fault: null,
message: null,
replicas: { desired, ready: desired, updated: desired },
state: "ready",
state_since: 0,
state_since: since,
waiting_on: null,
};
if (outcome === "ready") return ready;
return {
...ready,
action_needed: true,
fault: outcome.fault,
message: outcome.message,
replicas: { desired, ready: 0, updated: 0 },
state: outcome.state,
};
};
// A rolled-out pool reads as its old status `staleReads` times, as `updating` once, then as its outcome.
const poolView = (key: string, pool: CloudPool, read: boolean): Record<string, unknown> => {
const [region = "", name = ""] = key.split("/");
const state = poolStates.get(key);
const desired = pool.node_count;
const before = outcomeStatus(options.poolStatus ?? "ready", desired, 0);
let status = state === undefined ? before : outcomeStatus("ready", desired, state.since);
if (state?.rolling === true) {
if (read) state.reads += 1;
const outcome = options.poolOutcome ?? "ready";
if (state.reads <= 1) {
status = { ...status, replicas: { desired, ready: desired, updated: 0 }, state: "updating", waiting_on: "runners" };
} else if (outcome !== "ready") {
status = { ...status, action_needed: true, fault: outcome.fault, message: outcome.message, replicas: { desired, ready: 0, updated: 0 }, state: outcome.state };
const stale = options.staleReads ?? 0;
if (state.reads <= stale) {
status = before;
} else if (state.reads <= stale + 1) {
status = { ...status, replicas: { desired, ready: desired, updated: 0 }, state: "updating", state_since: state.since, waiting_on: "runners" };
} else {
state.rolling = false;
if (state.reads === stale + 2) state.since = Date.now();
status = outcomeStatus(options.poolOutcome ?? "ready", desired, state.since);
if (options.poolOutcome === undefined || options.poolOutcome === "ready") state.rolling = false;
}
}
return {
Expand Down Expand Up @@ -386,7 +408,7 @@ export async function startRivetCloudDouble(options: RivetCloudDoubleOptions): P
node_count: (body.node_count as number | undefined) ?? existing?.node_count ?? 1,
};
pools.set(key, pool);
poolStates.set(key, { reads: 0, rolling: true });
poolStates.set(key, { reads: 0, rolling: true, since: Date.now() });
send(response, 200, poolView(key, pool, false));
return;
}
Expand Down
Loading