From 95c1181d6a7db8d77bb6d4b1889b02ffe01eee36 Mon Sep 17 00:00:00 2001 From: Sree Narayanan Date: Sat, 14 Nov 2026 19:36:27 +0400 Subject: [PATCH] fix(deploy): retry dropped uploads and show upload progress A dropped connection during the image upload failed the deploy with a bug report link. The upload now gets 3 attempts, and so do Cloud API GET and PUT calls. POST calls are sent once, because a lost answer to a POST that already reached Rivet Cloud would create a second build if repeated. In a terminal, the spinner shows bytes sent and the upload rate. --- docs/deploy.mdx | 4 +- src/commands/deploy.ts | 23 ++++++- src/commands/rivet-cloud-api.ts | 100 ++++++++++++++++++++++------ tests/cli/orange-deploy.test.ts | 38 ++++++++++- tests/harness/rivet-cloud-double.ts | 19 ++++++ 5 files changed, 160 insertions(+), 24 deletions(-) diff --git a/docs/deploy.mdx b/docs/deploy.mdx index 6acbe08..4ed31ab 100644 --- a/docs/deploy.mdx +++ b/docs/deploy.mdx @@ -236,7 +236,9 @@ In CI, name the cluster with `--cluster` or `RIVET_CLUSTER`. The cluster token i to 5 GiB. 4. Orange registers a build, uploads the tarball to the build's upload URL, and waits until the build is `ready`. Rivet Cloud prepares a build in 3 to - 5 minutes. + 5 minutes. In a terminal, Orange shows the bytes sent and the upload rate. + When the connection drops, or Rivet Cloud or storage answers 408, 429, or + 5xx, Orange sends the request again, up to 3 attempts in total. 5. Orange rolls the runner pool out to the build. With an `orange login` session, it creates the pool if it does not exist, and prints `Creating pool default`. With a cluster token, the pool must exist. Every later deploy rolls diff --git a/src/commands/deploy.ts b/src/commands/deploy.ts index 44a95f8..d7f31f0 100644 --- a/src/commands/deploy.ts +++ b/src/commands/deploy.ts @@ -400,7 +400,7 @@ async function buildAndUpload( const md5 = await md5Base64(tarball); const created = await cloud.createBuild({ content_length: size, content_md5: md5, image: reference }); io.log(`Uploading build ${created.build_id} (${formatBytes(size)})…`); - await uploadFile(created.upload_url, tarball, size, md5); + await uploadWithProgress(io, theme, created.upload_url, tarball, size, md5); await cloud.completeBuild(created.build_id); io.log(`Preparing build ${created.build_id}. This takes a few minutes…`); await waitForBuild(cloud, theme, created.build_id); @@ -411,6 +411,27 @@ async function buildAndUpload( } } +async function uploadWithProgress(io: DeployCommandIo, theme: Theme, uploadUrl: string, tarball: string, size: number, md5: string): Promise { + const spinner = startSpinner(process.stderr, theme, `Uploading ${formatBytes(0)} of ${formatBytes(size)}…`); + let drawnAt = 0; + try { + await uploadFile(uploadUrl, tarball, size, md5, { + progress(sent, elapsedMs) { + const now = Date.now(); + if (now - drawnAt < 250 && sent < size) return; + drawnAt = now; + const rate = elapsedMs > 0 ? `, ${formatBytes((sent * 1000) / elapsedMs)}/s` : ""; + spinner.update(`Uploading ${formatBytes(sent)} of ${formatBytes(size)}${rate}…`); + }, + retry(reason, attempt, waitMs) { + spinner.interrupt(() => io.log(`Upload attempt ${attempt} failed. ${reason} Retrying in ${waitMs / 1000} s…`)); + }, + }); + } finally { + spinner.stop(); + } +} + async function waitForBuild(cloud: RivetCloudClient, theme: Theme, buildId: string): Promise { const deadline = Date.now() + buildTimeoutMs; const spinner = startSpinner(process.stderr, theme, `Preparing build ${buildId}…`); diff --git a/src/commands/rivet-cloud-api.ts b/src/commands/rivet-cloud-api.ts index c96b6ec..866ae9b 100644 --- a/src/commands/rivet-cloud-api.ts +++ b/src/commands/rivet-cloud-api.ts @@ -1,6 +1,7 @@ import { createReadStream } from "node:fs"; import { request as httpRequest, type IncomingMessage } from "node:http"; import { request as httpsRequest } from "node:https"; +import { setTimeout as delay } from "node:timers/promises"; import { z } from "zod"; import { ExpectedFailure } from "../cli/issue-report.js"; @@ -82,6 +83,9 @@ const apiErrorSchema = z.object({ }); const requestTimeoutMs = 60_000; +// A dropped connection or an overloaded server gets two more tries, as slow links drop large uploads. +const retryDelaysMs = [1_000, 3_000]; +const transientStatuses = new Set([408, 429, 500, 502, 503, 504]); export interface RivetCloudClient { completeBuild(buildId: string): Promise; @@ -123,12 +127,27 @@ export function createRivetCloudClient(options: RivetCloudOptions): RivetCloudCl method, signal: AbortSignal.timeout(requestTimeoutMs), }); + // A POST can reach Rivet Cloud and lose only its answer, and sending it again would create a + // second build or roll a secret twice. So only GET and PUT, which repeat safely, are retried. + const delays = method === "POST" ? [] : retryDelaysMs; // A network failure is the machine's or Rivet Cloud's, not a bug in Orange. const reach = async (token: string): Promise => { - try { - return await send(token); - } catch (error: unknown) { - throw new ExpectedFailure(`Failed to ${action}. Couldn't reach Rivet Cloud at ${url.origin}: ${causeOf(error)}`); + for (let attempt = 1; ; attempt += 1) { + let response: Response | undefined; + let failure: unknown; + try { + response = await send(token); + } catch (error: unknown) { + failure = error; + } + const wait = delays[attempt - 1]; + if (response !== undefined && (!transientStatuses.has(response.status) || wait === undefined)) return response; + if (wait === undefined) { + const tries = attempt > 1 ? ` after ${attempt} attempts` : ""; + throw new ExpectedFailure(`Failed to ${action}${tries}. Couldn't reach Rivet Cloud at ${url.origin}: ${causeOf(failure)}`); + } + await response?.body?.cancel(); + await delay(wait); } }; // A token error, such as an expired session, already says what failed and what to do. @@ -243,15 +262,52 @@ function apiError(action: string, cluster: string, status: number, text: string, return new Error(`Failed to ${action} (HTTP ${status}): ${reason}${fields.join("")}`); } -// Sends the file in one PUT with exactly `size` bytes and its Content-MD5, as the -// upload URL requires. The file streams from disk, so a large image is never in memory. -// `fetch` would send a streamed body chunked, without a Content-Length. -export function uploadFile(uploadUrl: string, path: string, size: number, md5: string): Promise { - const url = new URL(uploadUrl); +export interface UploadEvents { + // Bytes of the current attempt that left the file, and the attempt's age. + progress?(sent: number, elapsedMs: number): void; + retry?(reason: string, attempt: number, waitMs: number): void; +} + +// Sends the file in one PUT, and again to the same signed URL when the connection drops +// or storage answers with a transient error. A failure after the last attempt is the +// network's or storage's, not a bug in Orange. +export async function uploadFile(uploadUrl: string, path: string, size: number, md5: string, events: UploadEvents = {}): Promise { + for (let attempt = 1; ; attempt += 1) { + const result = await uploadOnce(new URL(uploadUrl), path, size, md5, events); + if (result === undefined) return; + const wait = retryDelaysMs[attempt - 1]; + if (!result.transient) throw new Error(`Failed to upload the image. ${result.reason}`); + if (wait === undefined) { + throw new ExpectedFailure(`Failed to upload the image after ${attempt} attempts. ${result.reason} Check the connection, then deploy again.`); + } + events.retry?.(result.reason, attempt, wait); + await delay(wait); + } +} + +// Sends exactly `size` bytes with their Content-MD5, as the upload URL requires. The file +// streams from disk, so a large image is never in memory. `fetch` would send a streamed +// body chunked, without a Content-Length. Resolves to the failure, or undefined on success. +function uploadOnce( + url: URL, + path: string, + size: number, + md5: string, + events: UploadEvents, +): Promise<{ reason: string; transient: boolean } | undefined> { const send = url.protocol === "https:" ? httpsRequest : httpRequest; - return new Promise((resolveUpload, rejectUpload) => { - const fail = (error: unknown): void => - rejectUpload(new Error(`Failed to upload the image: ${causeOf(error)}`)); + return new Promise((resolveUpload) => { + const file = createReadStream(path); + let settled = false; + const settle = (result: { reason: string; transient: boolean } | undefined): void => { + if (settled) return; + settled = true; + file.destroy(); + // A failed attempt must not keep its socket for the next one. + if (result !== undefined) request.destroy(); + resolveUpload(result); + }; + const fail = (error: unknown): void => settle({ reason: `The connection failed: ${sentence(causeOf(error))}`, transient: true }); const request = send( url, { headers: { "content-length": size, "content-md5": md5 }, method: "PUT" }, @@ -264,18 +320,20 @@ export function uploadFile(uploadUrl: string, path: string, size: number, md5: s response.on("error", fail); response.on("end", () => { const status = response.statusCode ?? 0; - if (status >= 200 && status < 300) { - resolveUpload(); - return; - } - rejectUpload(new Error(`Failed to upload the image. ${describeStorageError(status, body)}`)); + if (status >= 200 && status < 300) settle(undefined); + else settle({ reason: describeStorageError(status, body), transient: transientStatuses.has(status) }); }); }, ); request.on("error", fail); // A stalled upload fails instead of waiting forever. request.setTimeout(5 * 60_000, () => request.destroy(new Error("the upload stalled for 5 minutes"))); - const file = createReadStream(path); + const startedAt = Date.now(); + let sent = 0; + file.on("data", (chunk) => { + sent += chunk.length; + events.progress?.(sent, Date.now() - startedAt); + }); file.on("error", (error) => request.destroy(error)); file.pipe(request); }); @@ -287,11 +345,11 @@ function describeStorageError(status: number, body: string): string { const code = /([^<]*)<\/Code>/.exec(body)?.[1]; const message = /([^<]*)<\/Message>/.exec(body)?.[1]; if (code !== undefined || message !== undefined) { - return `Storage answered HTTP ${status} ${code ?? ""}: ${message ?? "no message"}`.replace(" :", ":"); + return sentence(`Storage answered HTTP ${status} ${code ?? ""}: ${message ?? "no message"}`.replace(" :", ":")); } - if (status === 408) return "Storage stopped receiving it and timed the upload out (HTTP 408). Check the connection, then deploy again."; + if (status === 408) return "Storage stopped receiving it and timed the upload out (HTTP 408)."; const text = body.trim(); - return `Storage answered HTTP ${status}${text.length > 0 && !text.startsWith("<") ? `: ${text.slice(0, 200)}` : ""}`; + return sentence(`Storage answered HTTP ${status}${text.length > 0 && !text.startsWith("<") ? `: ${text.slice(0, 200)}` : ""}`); } function causeOf(error: unknown): string { diff --git a/tests/cli/orange-deploy.test.ts b/tests/cli/orange-deploy.test.ts index 5880c9d..a5d36b8 100644 --- a/tests/cli/orange-deploy.test.ts +++ b/tests/cli/orange-deploy.test.ts @@ -122,13 +122,49 @@ test("an upload that storage times out fails with that reason, and its HTML erro await withDeploy({ uploadRejection: { html, status: 408 } }, async (cloud, project, env) => { const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url], env); assert.equal(code, 1, output); - assert.match(output, /Failed to upload the image\. Storage stopped receiving it and timed the upload out \(HTTP 408\)\. Check the connection, then deploy again\./); + assert.match(output, /Failed to upload the image after 3 attempts\. Storage stopped receiving it and timed the upload out \(HTTP 408\)\. Check the connection, then deploy again\./); assert.ok(!/DOCTYPE|Request\+?Timeout|%3Chtml/i.test(output), output); assert.ok(!cloud.requests.some((request) => request.endsWith("/complete")), cloud.requests.join("\n")); assert.equal(cloud.pools.size, 0); }); }); +test("an upload that loses its connection is sent again, and storage gets the whole image once", { skip: dockerSkip }, async () => { + await withDeploy({ dropConnections: { uploads: 1 } }, async (cloud, project, env, tag) => { + const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--tag", tag], env); + assert.equal(code, 0, output); + assert.match(output, /Upload attempt 1 failed\. The connection failed: .+ Retrying in 1 s…/); + assert.deepEqual( + cloud.uploads.map((upload) => ({ ...upload, bytes: undefined })), + [{ bytes: undefined, isImageTarball: true, matchesBuild: true }], + "storage received the whole image once", + ); + assert.ok(cloud.pools.get("iad/default")?.build_id?.startsWith("bld_") === true, output); + }); +}); + +test("a build registration whose answer is lost is not sent again, so Rivet Cloud holds one build", { skip: dockerSkip }, async () => { + await withDeploy({ dropConnections: { builds: 1 } }, async (cloud, project, env, tag) => { + const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--tag", tag], env); + assert.equal(code, 1, output); + assert.match(output, /Failed to register a build on cluster acme\. Couldn't reach Rivet Cloud at /); + assert.doesNotMatch(output, /Looks like a bug/); + assert.equal(cloud.requests.filter((request) => request.startsWith("POST ") && request.endsWith("/builds")).length, 1, cloud.requests.join("\n")); + assert.equal(cloud.pools.size, 0); + }); +}); + +test("an upload that loses its connection on every attempt fails with what to do next, and no bug report link", { skip: dockerSkip }, async () => { + await withDeploy({ dropConnections: { uploads: 3 } }, async (cloud, project, env, tag) => { + const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--tag", tag], env); + assert.equal(code, 1, output); + assert.match(output, /Failed to upload the image after 3 attempts\. The connection failed: .+ Check the connection, then deploy again\./); + assert.doesNotMatch(output, /Looks like a bug/); + assert.ok(!cloud.requests.some((request) => request.endsWith("/complete")), cloud.requests.join("\n")); + assert.equal(cloud.pools.size, 0); + }); +}); + test("a rejected token fails before anything is built and names where the token comes from", async () => { await withDeploy({}, async (cloud, project, env) => { const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url], { diff --git a/tests/harness/rivet-cloud-double.ts b/tests/harness/rivet-cloud-double.ts index 60265be..b8acad5 100644 --- a/tests/harness/rivet-cloud-double.ts +++ b/tests/harness/rivet-cloud-double.ts @@ -39,6 +39,9 @@ export interface RivetCloudDoubleOptions { token: string; // `token` is a cluster token, which Rivet Cloud lets update a pool but not create one. tokenIsClusterToken?: boolean; + // The first `builds` build registrations lose their answer after the build exists, and the + // first `uploads` uploads lose their connection after part of the body arrived. + dropConnections?: { builds?: number; uploads?: number }; // Cloud storage refuses the upload with this S3-style error. // `html` answers as the storage front end does, with a page instead of the XML error. uploadRejection?: { code: string; message: string; status: number } | { html: string; status: number }; @@ -199,7 +202,19 @@ export async function startRivetCloudDouble(options: RivetCloudDoubleOptions): P }; }; + const dropped = { builds: 0, uploads: 0 }; + const drops = (kind: "builds" | "uploads"): boolean => { + if (dropped[kind] >= (options.dropConnections?.[kind] ?? 0)) return false; + dropped[kind] += 1; + return true; + }; + const handleUpload = async (request: IncomingMessage, response: ServerResponse, buildId: string): Promise => { + if (drops("uploads")) { + await new Promise((resolveChunk) => request.once("data", resolveChunk)); + request.socket.destroy(); + return; + } const build = builds.get(buildId); const hash = createHash("md5"); let bytes = 0; @@ -333,6 +348,10 @@ export async function startRivetCloudDouble(options: RivetCloudDoubleOptions): P } const id = `bld_${randomUUID().slice(0, 8)}`; builds.set(id, { contentLength, contentMd5, failure: null, id, image, reads: 0, status: "uploading", uploaded: false }); + if (drops("builds")) { + request.socket.destroy(); + return; + } send(response, 201, { build_id: id, expires_at: new Date(Date.now() + 3_600_000).toISOString(),