Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion examples/site-deploy-ssg/__tests__/mock-server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,8 @@ export function createMockServer(): MockServer {
previewRefs: {},
commits: {},
api: async (query, variables) => handle(operationName(query), variables),
putObject: async (url, body, contentType) => {
putObject: async (url, body, contentType, _signal, _contentHash, headers) => {
assertSignedHeaders(url, headers);
const hash = url.slice(url.lastIndexOf('/') + 1);
hashes.add(hash);
server.storedKeys.push(`cas/sha256/${hash}`);
Expand All @@ -65,6 +66,7 @@ export function createMockServer(): MockServer {
key: file.key as string,
deduplicated,
uploadUrl: deduplicated ? null : `https://s3.test/put/${file.contentHash}`,
uploadHeaders: deduplicated ? null : signedHeaders(file),
};
}),
},
Expand Down Expand Up @@ -120,3 +122,20 @@ function operationName(query: string): string {
if (!match) throw new Error(`Could not read an operation from: ${query}`);
return match[1];
}

/** What the presigned-url plugin returns: the headers the URL is signed over. */
function signedHeaders(file: { contentHash: string; contentType: string }): Record<string, string> {
return {
'Content-Type': file.contentType,
'x-amz-checksum-sha256': Buffer.from(file.contentHash, 'hex').toString('base64'),
'x-amz-meta-upload-id': `file-${file.contentHash.slice(0, 8)}`,
};
}

/** A PUT without every signed header fails the signature, as S3 does. */
export function assertSignedHeaders(url: string, headers: Record<string, string> | undefined): void {
const hash = url.slice(url.lastIndexOf('/') + 1);
if (headers?.['x-amz-meta-upload-id'] !== `file-${hash.slice(0, 8)}`) {
throw new Error('SignatureDoesNotMatch: missing signed header: x-amz-meta-upload-id');
}
}
25 changes: 22 additions & 3 deletions packages/site-deploy/__tests__/deploy.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { deploySite } from '../src/deploy';
import type { DeployProgress } from '../src/types';
import type { DeployProgress, GraphQLExecutor, PutObject } from '../src/types';
import { DeployError } from '../src/types';
import { createMockServer, failingPut, file, hashOf, keyOf } from './mock-api';

Expand Down Expand Up @@ -36,6 +36,11 @@ describe('deploySite', () => {
size: 13,
});
expect(server.puts).toHaveLength(2);
expect(server.puts.find((put) => put.url.endsWith(hashOf('<h1>home</h1>')))?.headers).toEqual({
'Content-Type': 'text/html; charset=utf-8',
'x-amz-checksum-sha256': Buffer.from(hashOf('<h1>home</h1>'), 'hex').toString('base64'),
'x-amz-meta-upload-id': `file-${hashOf('<h1>home</h1>').slice(0, 8)}`,
});
expect(server.countOf('createSiteRelease')).toBe(1);
expect(server.countOf('updateSiteRelease')).toBe(0);
});
Expand Down Expand Up @@ -212,13 +217,27 @@ describe('deploySite', () => {
expect(server.activeCommitId).toBeNull();
});

it('refuses to upload a file the server returned no signed headers for', async () => {
const server = createMockServer();
const unsigned: GraphQLExecutor = async (query, variables) => {
const data = await server.api(query, variables);
for (const result of (data.uploadFiles as any)?.files ?? []) result.uploadHeaders = null;
return data;
};
await expect(
deploySite({ ...base, api: unsigned, putObject: server.putObject, source: build }),
).rejects.toMatchObject({ code: 'UPLOAD_FAILED' });
expect(server.puts).toHaveLength(0);
expect(server.countOf('createSiteRelease')).toBe(0);
});

it('retries a transient upload failure before giving up', async () => {
const server = createMockServer();
let attempts = 0;
const flaky = async (url: string, body: Uint8Array, contentType: string) => {
const flaky: PutObject = async (url, body, contentType, signal, contentHash, headers) => {
attempts += 1;
if (attempts === 1) throw new Error('502 bad gateway');
await server.putObject(url, body, contentType);
await server.putObject(url, body, contentType, signal, contentHash, headers);
};

const result = await deploySite({
Expand Down
25 changes: 22 additions & 3 deletions packages/site-deploy/__tests__/mock-api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ export interface MockServer {
api: GraphQLExecutor;
putObject: PutObject;
calls: MockCall[];
puts: { url: string; size: number; contentType: string }[];
puts: { url: string; size: number; contentType: string; headers?: Record<string, string> }[];
stored: Set<string>;
release: { id: string; commitId: string; storeId: string; manifest: ReleaseManifest } | null;
activeCommitId: string | null;
Expand Down Expand Up @@ -69,8 +69,9 @@ export function createMockServer(options: MockServerOptions = {}): MockServer {
}
return handle(operation, variables);
},
putObject: async (url, body, contentType) => {
server.puts.push({ url, size: body.byteLength, contentType });
putObject: async (url, body, contentType, _signal, _contentHash, headers) => {
assertSignedHeaders(url, headers);
server.puts.push({ url, size: body.byteLength, contentType, headers });
stored.add(url.slice(url.lastIndexOf('/') + 1));
},
countOf: (operation) => server.calls.filter((call) => call.operation === operation).length,
Expand All @@ -94,6 +95,7 @@ export function createMockServer(options: MockServerOptions = {}): MockServer {
key,
deduplicated,
uploadUrl: deduplicated ? null : `https://s3.test/put/${file.contentHash}`,
uploadHeaders: deduplicated ? null : signedHeaders(file),
};
}),
},
Expand Down Expand Up @@ -173,3 +175,20 @@ export function keyOf(body: string): string {
export function hashOf(body: string): string {
return hashBytes(new TextEncoder().encode(body));
}

/** What the presigned-url plugin returns: the headers the URL is signed over. */
function signedHeaders(file: { contentHash: string; contentType: string }): Record<string, string> {
return {
'Content-Type': file.contentType,
'x-amz-checksum-sha256': Buffer.from(file.contentHash, 'hex').toString('base64'),
'x-amz-meta-upload-id': `file-${file.contentHash.slice(0, 8)}`,
};
}

/** A PUT without every signed header fails the signature, as S3 does. */
export function assertSignedHeaders(url: string, headers: Record<string, string> | undefined): void {
const hash = url.slice(url.lastIndexOf('/') + 1);
if (headers?.['x-amz-meta-upload-id'] !== `file-${hash.slice(0, 8)}`) {
throw new Error('SignatureDoesNotMatch: missing signed header: x-amz-meta-upload-id');
}
}
19 changes: 10 additions & 9 deletions packages/site-deploy/src/cas-upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import { DeployError } from './types';
/** One entry of the bulk mutation's payload. */
interface BulkFilePayload {
uploadUrl: string | null;
uploadHeaders: Record<string, string> | null;
fileId: string;
key: string;
deduplicated: boolean;
Expand Down Expand Up @@ -50,8 +51,8 @@ export interface UploadCasResult {

/** The bytes the server does not have yet, as told by the bulk mutation. */
export interface CasDiff {
/** Files whose bytes must be PUT, paired with their presigned URL. */
missing: { file: HashedFile; uploadUrl: string }[];
/** Files whose bytes must be PUT, paired with their presigned URL and the headers it is signed over. */
missing: { file: HashedFile; uploadUrl: string; uploadHeaders: Record<string, string> }[];
/** Files the server already had. */
deduplicated: HashedFile[];
}
Expand Down Expand Up @@ -109,13 +110,13 @@ export async function diffCas(
deduplicated.push(file);
return;
}
if (!result.uploadUrl) {
if (!result.uploadUrl || !result.uploadHeaders) {
throw new DeployError(
'UPLOAD_FAILED',
`No upload URL for "${file.path}" and deduplicated=false`,
`No upload URL or upload headers for "${file.path}" and deduplicated=false`,
);
}
missing.push({ file, uploadUrl: result.uploadUrl });
missing.push({ file, uploadUrl: result.uploadUrl, uploadHeaders: result.uploadHeaders });
});
}

Expand All @@ -141,9 +142,9 @@ export async function uploadMissing(
throwIfAborted(options.signal);
const index = next++;
if (index >= diff.missing.length) return;
const { file, uploadUrl } = diff.missing[index];
const { file, uploadUrl, uploadHeaders } = diff.missing[index];
await withRetries(
() => put(uploadUrl, file.bytes, file.contentType, options.signal, file.hash),
() => put(uploadUrl, file.bytes, file.contentType, options.signal, file.hash, uploadHeaders),
options.retries,
file.path,
options.signal,
Expand All @@ -163,11 +164,11 @@ export async function uploadMissing(
};
}

const defaultPut: PutObject = async (url, body, contentType, signal, contentHash) => {
const defaultPut: PutObject = async (url, body, contentType, signal, contentHash, headers) => {
// A fresh copy: putToPresignedUrl takes a BodyInit, and a Uint8Array view
// over a pooled Buffer would send the whole pool.
const buffer = body.slice().buffer as ArrayBuffer;
await putToPresignedUrl(url, buffer, contentType, { signal, contentHash });
await putToPresignedUrl(url, buffer, contentType, { signal, contentHash, headers });
};

async function withRetries(
Expand Down
1 change: 1 addition & 0 deletions packages/site-deploy/src/documents.ts
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ export function buildBulkUploadMutation(mutation: string): string {
${mutation}(input: $input) {
files {
uploadUrl
uploadHeaders
fileId
key
deduplicated
Expand Down
5 changes: 5 additions & 0 deletions packages/site-deploy/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,11 @@ export type PutObject = (
signal?: AbortSignal,
/** Hex SHA-256 of `body`; the presigned URL is signed over it as `x-amz-checksum-sha256`. */
contentHash?: string,
/**
* The `uploadHeaders` the upload mutation returned: Content-Type and every
* header the URL is signed over (checksum, upload id). Send them as-is.
*/
headers?: Record<string, string>,
) => Promise<void>;

export interface DeploySiteOptions {
Expand Down
Loading