Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/deploy.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,9 @@ In CI, name the cluster with `--cluster` or `RIVET_CLUSTER`. The cluster token i
to 5 GiB.
4. Orange registers a build, uploads the tarball to the build's upload URL,
and waits until the build is `ready`. Rivet Cloud prepares a build in 3 to
5 minutes.
5 minutes. In a terminal, Orange shows the bytes sent and the upload rate.
When the connection drops, or Rivet Cloud or storage answers 408, 429, or
5xx, Orange sends the request again, up to 3 attempts in total.
5. Orange rolls the runner pool out to the build. With an `orange login` session, it
creates the pool if it does not exist, and prints `Creating pool default`. With a
cluster token, the pool must exist. Every later deploy rolls
Expand Down
23 changes: 22 additions & 1 deletion src/commands/deploy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ async function buildAndUpload(
const md5 = await md5Base64(tarball);
const created = await cloud.createBuild({ content_length: size, content_md5: md5, image: reference });
io.log(`Uploading build ${created.build_id} (${formatBytes(size)})…`);
await uploadFile(created.upload_url, tarball, size, md5);
await uploadWithProgress(io, theme, created.upload_url, tarball, size, md5);
await cloud.completeBuild(created.build_id);
io.log(`Preparing build ${created.build_id}. This takes a few minutes…`);
await waitForBuild(cloud, theme, created.build_id);
Expand All @@ -411,6 +411,27 @@ async function buildAndUpload(
}
}

async function uploadWithProgress(io: DeployCommandIo, theme: Theme, uploadUrl: string, tarball: string, size: number, md5: string): Promise<void> {
const spinner = startSpinner(process.stderr, theme, `Uploading ${formatBytes(0)} of ${formatBytes(size)}…`);
let drawnAt = 0;
try {
await uploadFile(uploadUrl, tarball, size, md5, {
progress(sent, elapsedMs) {
const now = Date.now();
if (now - drawnAt < 250 && sent < size) return;
drawnAt = now;
const rate = elapsedMs > 0 ? `, ${formatBytes((sent * 1000) / elapsedMs)}/s` : "";
spinner.update(`Uploading ${formatBytes(sent)} of ${formatBytes(size)}${rate}…`);
},
retry(reason, attempt, waitMs) {
spinner.interrupt(() => io.log(`Upload attempt ${attempt} failed. ${reason} Retrying in ${waitMs / 1000} s…`));
},
});
} finally {
spinner.stop();
}
}

async function waitForBuild(cloud: RivetCloudClient, theme: Theme, buildId: string): Promise<void> {
const deadline = Date.now() + buildTimeoutMs;
const spinner = startSpinner(process.stderr, theme, `Preparing build ${buildId}…`);
Expand Down
100 changes: 79 additions & 21 deletions src/commands/rivet-cloud-api.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { createReadStream } from "node:fs";
import { request as httpRequest, type IncomingMessage } from "node:http";
import { request as httpsRequest } from "node:https";
import { setTimeout as delay } from "node:timers/promises";
import { z } from "zod";

import { ExpectedFailure } from "../cli/issue-report.js";
Expand Down Expand Up @@ -82,6 +83,9 @@ const apiErrorSchema = z.object({
});

const requestTimeoutMs = 60_000;
// A dropped connection or an overloaded server gets two more tries, as slow links drop large uploads.
const retryDelaysMs = [1_000, 3_000];
const transientStatuses = new Set([408, 429, 500, 502, 503, 504]);

export interface RivetCloudClient {
completeBuild(buildId: string): Promise<void>;
Expand Down Expand Up @@ -123,12 +127,27 @@ export function createRivetCloudClient(options: RivetCloudOptions): RivetCloudCl
method,
signal: AbortSignal.timeout(requestTimeoutMs),
});
// A POST can reach Rivet Cloud and lose only its answer, and sending it again would create a
// second build or roll a secret twice. So only GET and PUT, which repeat safely, are retried.
const delays = method === "POST" ? [] : retryDelaysMs;
// A network failure is the machine's or Rivet Cloud's, not a bug in Orange.
const reach = async (token: string): Promise<Response> => {
try {
return await send(token);
} catch (error: unknown) {
throw new ExpectedFailure(`Failed to ${action}. Couldn't reach Rivet Cloud at ${url.origin}: ${causeOf(error)}`);
for (let attempt = 1; ; attempt += 1) {
let response: Response | undefined;
let failure: unknown;
try {
response = await send(token);
} catch (error: unknown) {
failure = error;
}
const wait = delays[attempt - 1];
if (response !== undefined && (!transientStatuses.has(response.status) || wait === undefined)) return response;
if (wait === undefined) {
const tries = attempt > 1 ? ` after ${attempt} attempts` : "";
throw new ExpectedFailure(`Failed to ${action}${tries}. Couldn't reach Rivet Cloud at ${url.origin}: ${causeOf(failure)}`);
}
await response?.body?.cancel();
await delay(wait);
}
};
// A token error, such as an expired session, already says what failed and what to do.
Expand Down Expand Up @@ -243,15 +262,52 @@ function apiError(action: string, cluster: string, status: number, text: string,
return new Error(`Failed to ${action} (HTTP ${status}): ${reason}${fields.join("")}`);
}

// Sends the file in one PUT with exactly `size` bytes and its Content-MD5, as the
// upload URL requires. The file streams from disk, so a large image is never in memory.
// `fetch` would send a streamed body chunked, without a Content-Length.
export function uploadFile(uploadUrl: string, path: string, size: number, md5: string): Promise<void> {
const url = new URL(uploadUrl);
export interface UploadEvents {
// Bytes of the current attempt that left the file, and the attempt's age.
progress?(sent: number, elapsedMs: number): void;
retry?(reason: string, attempt: number, waitMs: number): void;
}

// Sends the file in one PUT, and again to the same signed URL when the connection drops
// or storage answers with a transient error. A failure after the last attempt is the
// network's or storage's, not a bug in Orange.
export async function uploadFile(uploadUrl: string, path: string, size: number, md5: string, events: UploadEvents = {}): Promise<void> {
for (let attempt = 1; ; attempt += 1) {
const result = await uploadOnce(new URL(uploadUrl), path, size, md5, events);
if (result === undefined) return;
const wait = retryDelaysMs[attempt - 1];
if (!result.transient) throw new Error(`Failed to upload the image. ${result.reason}`);
if (wait === undefined) {
throw new ExpectedFailure(`Failed to upload the image after ${attempt} attempts. ${result.reason} Check the connection, then deploy again.`);
}
events.retry?.(result.reason, attempt, wait);
await delay(wait);
}
}

// Sends exactly `size` bytes with their Content-MD5, as the upload URL requires. The file
// streams from disk, so a large image is never in memory. `fetch` would send a streamed
// body chunked, without a Content-Length. Resolves to the failure, or undefined on success.
function uploadOnce(
url: URL,
path: string,
size: number,
md5: string,
events: UploadEvents,
): Promise<{ reason: string; transient: boolean } | undefined> {
const send = url.protocol === "https:" ? httpsRequest : httpRequest;
return new Promise((resolveUpload, rejectUpload) => {
const fail = (error: unknown): void =>
rejectUpload(new Error(`Failed to upload the image: ${causeOf(error)}`));
return new Promise((resolveUpload) => {
const file = createReadStream(path);
let settled = false;
const settle = (result: { reason: string; transient: boolean } | undefined): void => {
if (settled) return;
settled = true;
file.destroy();
// A failed attempt must not keep its socket for the next one.
if (result !== undefined) request.destroy();
resolveUpload(result);
};
const fail = (error: unknown): void => settle({ reason: `The connection failed: ${sentence(causeOf(error))}`, transient: true });
const request = send(
url,
{ headers: { "content-length": size, "content-md5": md5 }, method: "PUT" },
Expand All @@ -264,18 +320,20 @@ export function uploadFile(uploadUrl: string, path: string, size: number, md5: s
response.on("error", fail);
response.on("end", () => {
const status = response.statusCode ?? 0;
if (status >= 200 && status < 300) {
resolveUpload();
return;
}
rejectUpload(new Error(`Failed to upload the image. ${describeStorageError(status, body)}`));
if (status >= 200 && status < 300) settle(undefined);
else settle({ reason: describeStorageError(status, body), transient: transientStatuses.has(status) });
});
},
);
request.on("error", fail);
// A stalled upload fails instead of waiting forever.
request.setTimeout(5 * 60_000, () => request.destroy(new Error("the upload stalled for 5 minutes")));
const file = createReadStream(path);
const startedAt = Date.now();
let sent = 0;
file.on("data", (chunk) => {
sent += chunk.length;
events.progress?.(sent, Date.now() - startedAt);
});
file.on("error", (error) => request.destroy(error));
file.pipe(request);
});
Expand All @@ -287,11 +345,11 @@ function describeStorageError(status: number, body: string): string {
const code = /<Code>([^<]*)<\/Code>/.exec(body)?.[1];
const message = /<Message>([^<]*)<\/Message>/.exec(body)?.[1];
if (code !== undefined || message !== undefined) {
return `Storage answered HTTP ${status} ${code ?? ""}: ${message ?? "no message"}`.replace(" :", ":");
return sentence(`Storage answered HTTP ${status} ${code ?? ""}: ${message ?? "no message"}`.replace(" :", ":"));
}
if (status === 408) return "Storage stopped receiving it and timed the upload out (HTTP 408). Check the connection, then deploy again.";
if (status === 408) return "Storage stopped receiving it and timed the upload out (HTTP 408).";
const text = body.trim();
return `Storage answered HTTP ${status}${text.length > 0 && !text.startsWith("<") ? `: ${text.slice(0, 200)}` : ""}`;
return sentence(`Storage answered HTTP ${status}${text.length > 0 && !text.startsWith("<") ? `: ${text.slice(0, 200)}` : ""}`);
}

function causeOf(error: unknown): string {
Expand Down
38 changes: 37 additions & 1 deletion tests/cli/orange-deploy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,13 +122,49 @@ test("an upload that storage times out fails with that reason, and its HTML erro
await withDeploy({ uploadRejection: { html, status: 408 } }, async (cloud, project, env) => {
const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url], env);
assert.equal(code, 1, output);
assert.match(output, /Failed to upload the image\. Storage stopped receiving it and timed the upload out \(HTTP 408\)\. Check the connection, then deploy again\./);
assert.match(output, /Failed to upload the image after 3 attempts\. Storage stopped receiving it and timed the upload out \(HTTP 408\)\. Check the connection, then deploy again\./);
assert.ok(!/DOCTYPE|Request\+?Timeout|%3Chtml/i.test(output), output);
assert.ok(!cloud.requests.some((request) => request.endsWith("/complete")), cloud.requests.join("\n"));
assert.equal(cloud.pools.size, 0);
});
});

test("an upload that loses its connection is sent again, and storage gets the whole image once", { skip: dockerSkip }, async () => {
await withDeploy({ dropConnections: { uploads: 1 } }, async (cloud, project, env, tag) => {
const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--tag", tag], env);
assert.equal(code, 0, output);
assert.match(output, /Upload attempt 1 failed\. The connection failed: .+ Retrying in 1 s…/);
assert.deepEqual(
cloud.uploads.map((upload) => ({ ...upload, bytes: undefined })),
[{ bytes: undefined, isImageTarball: true, matchesBuild: true }],
"storage received the whole image once",
);
assert.ok(cloud.pools.get("iad/default")?.build_id?.startsWith("bld_") === true, output);
});
});

test("a build registration whose answer is lost is not sent again, so Rivet Cloud holds one build", { skip: dockerSkip }, async () => {
await withDeploy({ dropConnections: { builds: 1 } }, async (cloud, project, env, tag) => {
const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--tag", tag], env);
assert.equal(code, 1, output);
assert.match(output, /Failed to register a build on cluster acme\. Couldn't reach Rivet Cloud at /);
assert.doesNotMatch(output, /Looks like a bug/);
assert.equal(cloud.requests.filter((request) => request.startsWith("POST ") && request.endsWith("/builds")).length, 1, cloud.requests.join("\n"));
assert.equal(cloud.pools.size, 0);
});
});

test("an upload that loses its connection on every attempt fails with what to do next, and no bug report link", { skip: dockerSkip }, async () => {
await withDeploy({ dropConnections: { uploads: 3 } }, async (cloud, project, env, tag) => {
const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url, "--tag", tag], env);
assert.equal(code, 1, output);
assert.match(output, /Failed to upload the image after 3 attempts\. The connection failed: .+ Check the connection, then deploy again\./);
assert.doesNotMatch(output, /Looks like a bug/);
assert.ok(!cloud.requests.some((request) => request.endsWith("/complete")), cloud.requests.join("\n"));
assert.equal(cloud.pools.size, 0);
});
});

test("a rejected token fails before anything is built and names where the token comes from", async () => {
await withDeploy({}, async (cloud, project, env) => {
const { code, output } = await orangeDeploy(project, ["--cloud-api", cloud.url], {
Expand Down
19 changes: 19 additions & 0 deletions tests/harness/rivet-cloud-double.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ export interface RivetCloudDoubleOptions {
token: string;
// `token` is a cluster token, which Rivet Cloud lets update a pool but not create one.
tokenIsClusterToken?: boolean;
// The first `builds` build registrations lose their answer after the build exists, and the
// first `uploads` uploads lose their connection after part of the body arrived.
dropConnections?: { builds?: number; uploads?: number };
// Cloud storage refuses the upload with this S3-style error.
// `html` answers as the storage front end does, with a page instead of the XML error.
uploadRejection?: { code: string; message: string; status: number } | { html: string; status: number };
Expand Down Expand Up @@ -199,7 +202,19 @@ export async function startRivetCloudDouble(options: RivetCloudDoubleOptions): P
};
};

const dropped = { builds: 0, uploads: 0 };
const drops = (kind: "builds" | "uploads"): boolean => {
if (dropped[kind] >= (options.dropConnections?.[kind] ?? 0)) return false;
dropped[kind] += 1;
return true;
};

const handleUpload = async (request: IncomingMessage, response: ServerResponse, buildId: string): Promise<void> => {
if (drops("uploads")) {
await new Promise((resolveChunk) => request.once("data", resolveChunk));
request.socket.destroy();
return;
}
const build = builds.get(buildId);
const hash = createHash("md5");
let bytes = 0;
Expand Down Expand Up @@ -333,6 +348,10 @@ export async function startRivetCloudDouble(options: RivetCloudDoubleOptions): P
}
const id = `bld_${randomUUID().slice(0, 8)}`;
builds.set(id, { contentLength, contentMd5, failure: null, id, image, reads: 0, status: "uploading", uploaded: false });
if (drops("builds")) {
request.socket.destroy();
return;
}
send(response, 201, {
build_id: id,
expires_at: new Date(Date.now() + 3_600_000).toISOString(),
Expand Down
Loading