diff --git a/graphile/graphile-presigned-url-plugin/__tests__/s3-signer.integration.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/s3-signer.integration.test.ts index 52ec5d4f97..972c989bbd 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/s3-signer.integration.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/s3-signer.integration.test.ts @@ -11,8 +11,11 @@ import { S3Client } from '@aws-sdk/client-s3'; import { createS3Bucket } from '@constructive-io/s3-utils'; +import { createHash } from 'crypto'; import { + CHECKSUM_SHA256_HEADER, + checksumSha256Base64, deleteS3Object, generatePresignedGetUrl, generatePresignedPutUrl, @@ -71,17 +74,23 @@ async function uploadToPresignedUrl( url: string, body: string, contentType: string, + extraHeaders: Record = {}, ): Promise { return fetch(url, { method: 'PUT', headers: { 'Content-Type': contentType, 'Content-Length': String(Buffer.byteLength(body)), + ...extraHeaders, }, body, }); } +function sha256Hex(content: string): string { + return createHash('sha256').update(content).digest('hex'); +} + /** * Download content from a presigned GET URL using native fetch. */ @@ -145,6 +154,68 @@ describe('s3-signer integration (RustFS)', () => { }); }); + describe('checksum-bound presigned PUT (content_hash binds the bytes)', () => { + const contentType = 'text/plain'; + const real = 'the bytes this hash names'; + const forged = 'FORGED bytes, same length'; // same Content-Length as `real` + const hash = sha256Hex(real); + + beforeAll(() => { + expect(Buffer.byteLength(forged)).toBe(Buffer.byteLength(real)); + }); + + it('signs x-amz-checksum-sha256 as a header, not a query parameter', async () => { + const url = await generatePresignedPutUrl(s3Config, 'checksum-shape.txt', contentType, Buffer.byteLength(real), 900, hash); + const params = new URL(url).searchParams; + expect(params.get('X-Amz-SignedHeaders')).toContain(CHECKSUM_SHA256_HEADER); + expect(params.has(CHECKSUM_SHA256_HEADER)).toBe(false); + }); + + it('accepts the bytes that hash to the content hash', async () => { + const key = 'checksum-ok.txt'; + const url = await generatePresignedPutUrl(s3Config, key, contentType, Buffer.byteLength(real), 900, hash); + const res = await uploadToPresignedUrl(url, real, contentType, { + [CHECKSUM_SHA256_HEADER]: checksumSha256Base64(hash), + }); + expect(res.status).toBe(200); + + const head = await headObject(s3Config, key); + expect(head?.checksumSha256).toBe(checksumSha256Base64(hash)); + }); + + it('rejects same-length bytes that do not hash to it (BadDigest)', async () => { + const key = 'checksum-forged.txt'; + const url = await generatePresignedPutUrl(s3Config, key, contentType, Buffer.byteLength(real), 900, hash); + const res = await uploadToPresignedUrl(url, forged, contentType, { + [CHECKSUM_SHA256_HEADER]: checksumSha256Base64(hash), + }); + expect(res.status).toBe(400); + expect(await res.text()).toContain('BadDigest'); + expect(await headObject(s3Config, key)).toBeNull(); + }); + + it('rejects a PUT that omits the checksum header (signature covers it)', async () => { + const key = 'checksum-missing.txt'; + const url = await generatePresignedPutUrl(s3Config, key, contentType, Buffer.byteLength(real), 900, hash); + const res = await uploadToPresignedUrl(url, forged, contentType); + expect(res.status).toBe(403); + expect(await headObject(s3Config, key)).toBeNull(); + }); + + it('cannot replace a confirmed object through the same URL with different bytes', async () => { + const key = 'checksum-replay.txt'; + const url = await generatePresignedPutUrl(s3Config, key, contentType, Buffer.byteLength(real), 900, hash); + const headers = { [CHECKSUM_SHA256_HEADER]: checksumSha256Base64(hash) }; + expect((await uploadToPresignedUrl(url, real, contentType, headers)).status).toBe(200); + + expect((await uploadToPresignedUrl(url, forged, contentType, headers)).status).toBe(400); + expect((await uploadToPresignedUrl(url, forged, contentType)).status).toBe(403); + + const getUrl = await generatePresignedGetUrl(s3Config, key); + expect((await downloadFromPresignedUrl(getUrl)).body).toBe(real); + }); + }); + describe('headObject', () => { const HEAD_KEY = 'test-head-object.json'; const HEAD_CONTENT = JSON.stringify({ test: true }); diff --git a/graphile/graphile-presigned-url-plugin/src/index.ts b/graphile/graphile-presigned-url-plugin/src/index.ts index a471be8ae9..7852a54f21 100644 --- a/graphile/graphile-presigned-url-plugin/src/index.ts +++ b/graphile/graphile-presigned-url-plugin/src/index.ts @@ -38,7 +38,15 @@ export { export { createPresignedUrlPlugin,PresignedUrlPlugin } from './plugin'; export { PresignedUrlPreset } from './preset'; export { describeS3Failure, s3FailureError } from './s3-failure'; -export { copyS3Object, deleteS3Object, generatePresignedGetUrl, generatePresignedPutUrl, headObject } from './s3-signer'; +export { + CHECKSUM_SHA256_HEADER, + checksumSha256Base64, + copyS3Object, + deleteS3Object, + generatePresignedGetUrl, + generatePresignedPutUrl, + headObject, +} from './s3-signer'; export { clearBucketCache, clearStorageModuleCache, getBucketConfig, loadAllStorageModules,resolveStorageConfigFromCodec, resolveStorageModuleByFileId } from './storage-module-cache'; export type { BucketConfig, diff --git a/graphile/graphile-presigned-url-plugin/src/plugin.ts b/graphile/graphile-presigned-url-plugin/src/plugin.ts index 3b404575c7..2d1754372f 100644 --- a/graphile/graphile-presigned-url-plugin/src/plugin.ts +++ b/graphile/graphile-presigned-url-plugin/src/plugin.ts @@ -829,13 +829,16 @@ export async function processSingleFile( path: derivedPath, }); - // Generate presigned PUT URL + // Generate presigned PUT URL, signed over the client's SHA-256 so the store + // only accepts bytes that hash to the content hash the row (and, for + // content-addressed buckets, the key and any later dedup hit) is named by. const uploadUrl = await generatePresignedPutUrl( s3ForDb, s3Key, contentType, size, storageConfig.uploadUrlExpirySeconds, + contentHash, ); const expiresAt = new Date(Date.now() + storageConfig.uploadUrlExpirySeconds * 1000).toISOString(); diff --git a/graphile/graphile-presigned-url-plugin/src/s3-signer.ts b/graphile/graphile-presigned-url-plugin/src/s3-signer.ts index 15346cab01..96e13ebb9d 100644 --- a/graphile/graphile-presigned-url-plugin/src/s3-signer.ts +++ b/graphile/graphile-presigned-url-plugin/src/s3-signer.ts @@ -13,6 +13,20 @@ import type { S3Config } from './types'; const log = new Logger('graphile-presigned-url:s3'); +/** The header a checksum-bound presigned PUT must carry (RFC 4648 base64 of the raw digest). */ +export const CHECKSUM_SHA256_HEADER = 'x-amz-checksum-sha256'; + +/** + * Convert a hex-encoded SHA-256 (the `content_hash` clients send) to the + * base64-of-raw-digest form S3 expects in `x-amz-checksum-sha256`. + */ +export function checksumSha256Base64(contentHashHex: string): string { + if (!/^[0-9a-f]{64}$/i.test(contentHashHex)) { + throw new Error('INVALID_CONTENT_HASH_FORMAT: must be a 64-char hex SHA-256'); + } + return Buffer.from(contentHashHex, 'hex').toString('base64'); +} + /** * Generate a presigned PUT URL for uploading a file to S3. * @@ -20,11 +34,18 @@ const log = new Logger('graphile-presigned-url:s3'); * content-length via the signature. The client MUST use these exact values * when performing the PUT request. * + * When `contentHash` is given, the SHA-256 is signed into the request as the + * `x-amz-checksum-sha256` header (kept out of the query string, where stores + * ignore it). The store then rejects any body that does not hash to it + * (`BadDigest`), and a PUT without the header fails the signature. This is + * what makes a content-addressed key actually hold the content it names. + * * @param s3Config - S3 client and bucket configuration * @param key - S3 object key (content hash or UUID) * @param contentType - MIME type (locked into the signature) * @param contentLength - File size in bytes (locked into the signature) * @param expiresIn - URL expiry in seconds (default: 900 = 15 minutes) + * @param contentHash - Hex SHA-256 the uploaded bytes must hash to * @returns Presigned PUT URL */ export async function generatePresignedPutUrl( @@ -33,17 +54,22 @@ export async function generatePresignedPutUrl( contentType: string, contentLength: number, expiresIn: number = 900, + contentHash?: string, ): Promise { const command = new PutObjectCommand({ Bucket: s3Config.bucket, Key: key, ContentType: contentType, ContentLength: contentLength, + ...(contentHash ? { ChecksumSHA256: checksumSha256Base64(contentHash) } : {}), }); let url: string; try { - url = await getSignedUrl(s3Config.client as any, command, { expiresIn }); + url = await getSignedUrl(s3Config.client as any, command, { + expiresIn, + ...(contentHash ? { unhoistableHeaders: new Set([CHECKSUM_SHA256_HEADER]) } : {}), + }); } catch (err) { throw s3FailureError( 'PRESIGN_PUT_FAILED', @@ -51,7 +77,9 @@ export async function generatePresignedPutUrl( err, ); } - log.debug(`Generated presigned PUT URL for key=${key}, contentType=${contentType}, expires=${expiresIn}s`); + log.debug( + `Generated presigned PUT URL for key=${key}, contentType=${contentType}, checksum=${contentHash ? 'sha256' : 'none'}, expires=${expiresIn}s`, + ); return url; } @@ -148,6 +176,11 @@ export async function copyS3Object( /** * Check if an object exists in S3 and optionally verify its content-type. * + * Asks the store for the object's checksum (`ChecksumMode: ENABLED`); it is + * returned as `checksumSha256` (base64 of the raw digest) when the object was + * written through a checksum-bound PUT, so callers can verify the bytes + * behind a key still hash to the content hash it names. + * * @param s3Config - S3 client and bucket configuration * @param key - S3 object key * @param expectedContentType - Expected MIME type (optional) @@ -157,12 +190,13 @@ export async function headObject( s3Config: S3Config, key: string, expectedContentType?: string, -): Promise<{ contentType?: string; contentLength?: number } | null> { +): Promise<{ contentType?: string; contentLength?: number; checksumSha256?: string } | null> { try { const response = await s3Config.client.send( new HeadObjectCommand({ Bucket: s3Config.bucket, Key: key, + ChecksumMode: 'ENABLED', }), ); @@ -175,6 +209,7 @@ export async function headObject( return { contentType: response.ContentType, contentLength: response.ContentLength, + ...(response.ChecksumSHA256 ? { checksumSha256: response.ChecksumSHA256 } : {}), }; } catch (e: any) { if (e.name === 'NotFound' || e.$metadata?.httpStatusCode === 404) { diff --git a/graphql/query/src/storage/transport.ts b/graphql/query/src/storage/transport.ts index daddaa7d11..11b7a7c004 100644 --- a/graphql/query/src/storage/transport.ts +++ b/graphql/query/src/storage/transport.ts @@ -23,12 +23,17 @@ export interface StorageFile { export interface StorageTransport { /** SHA-256 the file contents, hex-encoded */ hashFile(file: StorageFile): Promise; - /** PUT the bytes to a presigned URL */ + /** + * PUT the bytes to a presigned URL. `contentHash` is the hex SHA-256 the + * upload was requested with; the URL is signed over it as + * `x-amz-checksum-sha256`, so the transport must send it. + */ putObject( url: string, body: ArrayBuffer, contentType: string, signal?: AbortSignal, + contentHash?: string, ): Promise; } diff --git a/graphql/query/src/storage/upload.ts b/graphql/query/src/storage/upload.ts index 7a94865ae6..f94077288f 100644 --- a/graphql/query/src/storage/upload.ts +++ b/graphql/query/src/storage/upload.ts @@ -114,7 +114,7 @@ export async function uploadToSurface( } checkAborted(signal); - await transport.putObject(payload.uploadUrl, await file.arrayBuffer(), contentType, signal); + await transport.putObject(payload.uploadUrl, await file.arrayBuffer(), contentType, signal, contentHash); return toResult(payload); } diff --git a/graphql/server-test/__tests__/db-scope-upload.integration.test.ts b/graphql/server-test/__tests__/db-scope-upload.integration.test.ts index 1653e51305..69cd29efc0 100644 --- a/graphql/server-test/__tests__/db-scope-upload.integration.test.ts +++ b/graphql/server-test/__tests__/db-scope-upload.integration.test.ts @@ -204,7 +204,7 @@ describe('database-scope upload surface', () => { }); it('accepts the PUT, into a physical bucket recorded on the tenant row', async () => { - const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType); + const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { contentHash }); expect(putRes.ok).toBe(true); const stored = await pg.query( diff --git a/graphql/server-test/__tests__/upload-columns.integration.test.ts b/graphql/server-test/__tests__/upload-columns.integration.test.ts index 0f4ce9cf3b..11968b5f5f 100644 --- a/graphql/server-test/__tests__/upload-columns.integration.test.ts +++ b/graphql/server-test/__tests__/upload-columns.integration.test.ts @@ -152,7 +152,7 @@ describe('image / upload / attachment columns', () => { }); expect(uploaded.body.errors).toBeUndefined(); const { fileId, key, uploadUrl } = uploaded.body.data.uploadAppFile; - expect((await putToPresignedUrl(uploadUrl, content, 'text/plain')).ok).toBe(true); + expect((await putToPresignedUrl(uploadUrl, content, 'text/plain', { contentHash })).ok).toBe(true); // The confirm worker is not part of this fixture; it is what moves a row // from `requested` to `uploaded` once it has inspected the object. await pg.query(`UPDATE "${storageSchema}".app_files SET status = 'uploaded' WHERE id = $1`, [fileId]); diff --git a/graphql/server-test/__tests__/upload.integration.test.ts b/graphql/server-test/__tests__/upload.integration.test.ts index 69ddd697da..3f86308ae2 100644 --- a/graphql/server-test/__tests__/upload.integration.test.ts +++ b/graphql/server-test/__tests__/upload.integration.test.ts @@ -425,7 +425,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { }); it('should accept a PUT to the presigned URL', async () => { - const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType); + const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { contentHash }); expect(putRes.ok).toBe(true); await confirmUploaded(contentHash); }); @@ -468,7 +468,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { }); it('should accept a PUT to the presigned URL', async () => { - const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType); + const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { contentHash }); expect(putRes.ok).toBe(true); }); }); @@ -524,7 +524,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { expect(retry.uploadUrl).toBeTruthy(); expect(retry.fileId).not.toBe(first.fileId); - const putRes = await putToPresignedUrl(retry.uploadUrl, fileContent, 'text/plain'); + const putRes = await putToPresignedUrl(retry.uploadUrl, fileContent, 'text/plain', { contentHash }); expect(putRes.ok).toBe(true); await confirmUploaded(contentHash); @@ -869,7 +869,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { expect(payload.fileId).toBeTruthy(); expect(payload.uploadUrl).toBeTruthy(); - const putRes = await putToPresignedUrl(payload.uploadUrl, fileContent, 'text/plain'); + const putRes = await putToPresignedUrl(payload.uploadUrl, fileContent, 'text/plain', { contentHash }); expect(putRes.ok).toBe(true); }); diff --git a/packages/bucket-provisioner/__tests__/cors.test.ts b/packages/bucket-provisioner/__tests__/cors.test.ts index 25374ecaee..9c6dcaabab 100644 --- a/packages/bucket-provisioner/__tests__/cors.test.ts +++ b/packages/bucket-provisioner/__tests__/cors.test.ts @@ -23,6 +23,7 @@ describe('buildUploadCorsRules', () => { expect(rule.allowedHeaders).toContain('Content-Type'); expect(rule.allowedHeaders).toContain('Content-Length'); expect(rule.allowedHeaders).toContain('Authorization'); + expect(rule.allowedHeaders).toContain('x-amz-checksum-sha256'); }); it('exposes ETag and Content-Length', () => { @@ -74,6 +75,7 @@ describe('buildPrivateCorsRules', () => { expect(rule.allowedHeaders).toContain('Content-Type'); expect(rule.allowedHeaders).toContain('Content-Length'); expect(rule.allowedHeaders).toContain('Authorization'); + expect(rule.allowedHeaders).toContain('x-amz-checksum-sha256'); }); it('uses default maxAgeSeconds of 3600', () => { diff --git a/packages/bucket-provisioner/src/cors.ts b/packages/bucket-provisioner/src/cors.ts index 5176edd67f..d3a632854b 100644 --- a/packages/bucket-provisioner/src/cors.ts +++ b/packages/bucket-provisioner/src/cors.ts @@ -38,6 +38,7 @@ export function buildUploadCorsRules( 'Content-Length', 'Content-MD5', 'x-amz-content-sha256', + 'x-amz-checksum-sha256', 'x-amz-date', 'x-amz-security-token', 'Authorization', @@ -81,6 +82,7 @@ export function buildPrivateCorsRules( 'Content-Length', 'Content-MD5', 'x-amz-content-sha256', + 'x-amz-checksum-sha256', 'x-amz-date', 'x-amz-security-token', 'Authorization', diff --git a/packages/site-deploy/src/cas-upload.ts b/packages/site-deploy/src/cas-upload.ts index 395c003dfe..34b6fb2861 100644 --- a/packages/site-deploy/src/cas-upload.ts +++ b/packages/site-deploy/src/cas-upload.ts @@ -143,7 +143,7 @@ export async function uploadMissing( if (index >= diff.missing.length) return; const { file, uploadUrl } = diff.missing[index]; await withRetries( - () => put(uploadUrl, file.bytes, file.contentType, options.signal), + () => put(uploadUrl, file.bytes, file.contentType, options.signal, file.hash), options.retries, file.path, options.signal, @@ -163,11 +163,11 @@ export async function uploadMissing( }; } -const defaultPut: PutObject = async (url, body, contentType, signal) => { +const defaultPut: PutObject = async (url, body, contentType, signal, contentHash) => { // A fresh copy: putToPresignedUrl takes a BodyInit, and a Uint8Array view // over a pooled Buffer would send the whole pool. const buffer = body.slice().buffer as ArrayBuffer; - await putToPresignedUrl(url, buffer, contentType, signal); + await putToPresignedUrl(url, buffer, contentType, { signal, contentHash }); }; async function withRetries( diff --git a/packages/site-deploy/src/types.ts b/packages/site-deploy/src/types.ts index 8fd64180dc..f557b61987 100644 --- a/packages/site-deploy/src/types.ts +++ b/packages/site-deploy/src/types.ts @@ -104,6 +104,8 @@ export type PutObject = ( body: Uint8Array, contentType: string, signal?: AbortSignal, + /** Hex SHA-256 of `body`; the presigned URL is signed over it as `x-amz-checksum-sha256`. */ + contentHash?: string, ) => Promise; export interface DeploySiteOptions { diff --git a/packages/upload-client/__tests__/checksum.test.ts b/packages/upload-client/__tests__/checksum.test.ts new file mode 100644 index 0000000000..dbf59310a2 --- /dev/null +++ b/packages/upload-client/__tests__/checksum.test.ts @@ -0,0 +1,51 @@ +import { createHash } from 'crypto'; + +import { CHECKSUM_SHA256_HEADER, presignedPutHeaders, sha256HexOfBody, sha256HexToBase64 } from '../src/checksum'; + +describe('sha256HexToBase64', () => { + it('encodes the raw digest bytes, not the hex string', () => { + const hex = createHash('sha256').update('hello').digest('hex'); + const expected = createHash('sha256').update('hello').digest('base64'); + expect(sha256HexToBase64(hex)).toBe(expected); + expect(sha256HexToBase64(hex)).toHaveLength(44); + expect(sha256HexToBase64(hex)).not.toBe(Buffer.from(hex).toString('base64')); + }); + + it('rejects anything that is not a 64-char hex digest', () => { + expect(() => sha256HexToBase64('abc')).toThrow(/Invalid SHA-256/); + expect(() => sha256HexToBase64('z'.repeat(64))).toThrow(/Invalid SHA-256/); + }); +}); + +describe('presignedPutHeaders', () => { + const hex = 'a'.repeat(64); + + it('adds the signed checksum header when the hash is known', () => { + expect(presignedPutHeaders('text/plain', hex)).toEqual({ + 'Content-Type': 'text/plain', + [CHECKSUM_SHA256_HEADER]: sha256HexToBase64(hex), + }); + }); + + it('sends only Content-Type when there is no hash', () => { + expect(presignedPutHeaders('text/plain')).toEqual({ 'Content-Type': 'text/plain' }); + }); +}); + +describe('sha256HexOfBody', () => { + const expected = createHash('sha256').update('hello').digest('hex'); + + it('hashes in-memory bodies of every shape the same way', async () => { + const bytes = new TextEncoder().encode('hello'); + expect(await sha256HexOfBody('hello')).toBe(expected); + expect(await sha256HexOfBody(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength))).toBe(expected); + expect(await sha256HexOfBody(bytes)).toBe(expected); + expect(await sha256HexOfBody(Buffer.from('hello'))).toBe(expected); + expect(await sha256HexOfBody(new Blob(['hello']))).toBe(expected); + }); + + it('declines a body it cannot read up front', async () => { + expect(await sha256HexOfBody(new ReadableStream())).toBeUndefined(); + expect(await sha256HexOfBody(undefined)).toBeUndefined(); + }); +}); diff --git a/packages/upload-client/__tests__/upload.test.ts b/packages/upload-client/__tests__/upload.test.ts index 19dd3f3f7c..93358d2020 100644 --- a/packages/upload-client/__tests__/upload.test.ts +++ b/packages/upload-client/__tests__/upload.test.ts @@ -105,7 +105,11 @@ describe('uploadFile', () => { 'https://s3.example.com/presigned-put-url', expect.objectContaining({ method: 'PUT', - headers: { 'Content-Type': 'text/plain' }, + headers: { + 'Content-Type': 'text/plain', + // base64(sha256("hello world")): the presigned PUT is signed over it + 'x-amz-checksum-sha256': 'uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=', + }, }), ); }); diff --git a/packages/upload-client/src/checksum.ts b/packages/upload-client/src/checksum.ts new file mode 100644 index 0000000000..6d3103a696 --- /dev/null +++ b/packages/upload-client/src/checksum.ts @@ -0,0 +1,56 @@ +/** + * The presigned PUT URL the server hands out is signed over an + * `x-amz-checksum-sha256` header, so the object store only accepts bytes that + * hash to the `contentHash` the upload was requested with. Clients already + * compute that hex SHA-256; this turns it into the header the store expects. + */ + +import { sha256 } from '@decryption/hashes/sha2'; +import { bytesToHex, hexToBytes } from '@decryption/hashes/utils'; + +/** Header the presigned PUT is signed over. */ +export const CHECKSUM_SHA256_HEADER = 'x-amz-checksum-sha256'; + +/** Hex SHA-256 → base64 of the raw 32-byte digest (the S3 checksum encoding). */ +export function sha256HexToBase64(contentHash: string): string { + if (!/^[0-9a-f]{64}$/i.test(contentHash)) { + throw new Error(`Invalid SHA-256 hex content hash: ${contentHash}`); + } + const bytes = hexToBytes(contentHash.toLowerCase()); + if (typeof btoa === 'function') { + let binary = ''; + for (const b of bytes) binary += String.fromCharCode(b); + return btoa(binary); + } + return Buffer.from(bytes).toString('base64'); +} + +/** Headers for a presigned PUT: `Content-Type` plus the signed checksum when known. */ +export function presignedPutHeaders(contentType: string, contentHash?: string): Record { + return contentHash + ? { 'Content-Type': contentType, [CHECKSUM_SHA256_HEADER]: sha256HexToBase64(contentHash) } + : { 'Content-Type': contentType }; +} + +/** + * Hex SHA-256 of a PUT body when it is fully in memory (string, ArrayBuffer, + * typed array, or anything with `arrayBuffer()` such as a Blob/File). + * Returns undefined for bodies that cannot be hashed up front (streams), in + * which case the caller must supply the hash itself. + */ +export async function sha256HexOfBody(body: unknown): Promise { + if (typeof body === 'string') { + return bytesToHex(sha256(new TextEncoder().encode(body))); + } + if (body instanceof ArrayBuffer) { + return bytesToHex(sha256(new Uint8Array(body))); + } + if (ArrayBuffer.isView(body)) { + return bytesToHex(sha256(new Uint8Array(body.buffer, body.byteOffset, body.byteLength))); + } + if (body && typeof (body as { arrayBuffer?: unknown }).arrayBuffer === 'function') { + const buffer = await (body as { arrayBuffer(): Promise }).arrayBuffer(); + return bytesToHex(sha256(new Uint8Array(buffer))); + } + return undefined; +} diff --git a/packages/upload-client/src/index.ts b/packages/upload-client/src/index.ts index d5e2a40f66..013b2e8381 100644 --- a/packages/upload-client/src/index.ts +++ b/packages/upload-client/src/index.ts @@ -37,6 +37,8 @@ export { hashFile, hashFileChunked } from './hash'; export { hashContent } from './hash-content'; // Presigned URL helpers +export { CHECKSUM_SHA256_HEADER, presignedPutHeaders, sha256HexOfBody, sha256HexToBase64 } from './checksum'; +export type { PutToPresignedUrlOptions } from './put'; export { fetchFromUrl,putToPresignedUrl } from './put'; // Orchestrator diff --git a/packages/upload-client/src/put.ts b/packages/upload-client/src/put.ts index 274033966b..9153cfa228 100644 --- a/packages/upload-client/src/put.ts +++ b/packages/upload-client/src/put.ts @@ -20,10 +20,33 @@ import { createFetch } from '@constructive-io/fetch'; +import { presignedPutHeaders, sha256HexOfBody } from './checksum'; import { UploadError } from './types'; const fetch = createFetch(); +export interface PutToPresignedUrlOptions { + signal?: AbortSignal; + /** + * Hex SHA-256 of `body`. Sent as `x-amz-checksum-sha256`, which the + * server-issued presigned URL is signed over: the store rejects bytes that + * do not hash to it, and a PUT without it fails the signature. When omitted + * it is computed from `body` if the body is in memory (string, ArrayBuffer, + * typed array, Blob); a streamed body must pass it explicitly. + */ + contentHash?: string; +} + +function normalizePutOptions( + signalOrOptions?: AbortSignal | PutToPresignedUrlOptions, +): PutToPresignedUrlOptions { + if (!signalOrOptions) return {}; + if (typeof (signalOrOptions as AbortSignal).aborted === 'boolean') { + return { signal: signalOrOptions as AbortSignal }; + } + return signalOrOptions as PutToPresignedUrlOptions; +} + /** * PUT content to a presigned S3 URL. * @@ -41,12 +64,14 @@ export async function putToPresignedUrl( url: string, body: BodyInit | string, contentType: string, - signal?: AbortSignal, + signalOrOptions?: AbortSignal | PutToPresignedUrlOptions, ): Promise { + const { signal, contentHash } = normalizePutOptions(signalOrOptions); try { + const checksumSource = contentHash ?? (await sha256HexOfBody(body)); const response = await fetch(url, { method: 'PUT', - headers: { 'Content-Type': contentType }, + headers: presignedPutHeaders(contentType, checksumSource), body: body as BodyInit, signal, }); diff --git a/packages/upload-client/src/storage-transport.ts b/packages/upload-client/src/storage-transport.ts index 70518a517e..535541f54c 100644 --- a/packages/upload-client/src/storage-transport.ts +++ b/packages/upload-client/src/storage-transport.ts @@ -13,19 +13,24 @@ import type { FileInput } from './types'; export interface StorageTransportPort { hashFile(file: FileInput): Promise; + /** + * PUT the bytes to a presigned URL. `contentHash` is the hex SHA-256 the + * URL was requested with; it is sent as the signed `x-amz-checksum-sha256`. + */ putObject( url: string, body: ArrayBuffer, contentType: string, signal?: AbortSignal, + contentHash?: string, ): Promise; } -/** The presigned-URL transport: SHA-256 hashing plus a presigned PUT. */ +/** The presigned-URL transport: SHA-256 hashing plus a checksum-bound presigned PUT. */ export const storageTransport: StorageTransportPort = { hashFile: (file) => hashFile(file), - putObject: async (url, body, contentType, signal) => { - await putToPresignedUrl(url, body, contentType, signal); + putObject: async (url, body, contentType, signal, contentHash) => { + await putToPresignedUrl(url, body, contentType, { signal, contentHash }); }, }; diff --git a/packages/upload-client/src/upload.ts b/packages/upload-client/src/upload.ts index 7b2b0d316d..11dd1db5f6 100644 --- a/packages/upload-client/src/upload.ts +++ b/packages/upload-client/src/upload.ts @@ -7,6 +7,7 @@ * Each step is a pure function — this module just wires them together. */ +import { presignedPutHeaders } from './checksum'; import { hashFile } from './hash'; import { putToPresignedUrl } from './put'; import { buildRequestUploadUrlQuery, DEFAULT_BUCKET_QUERY_FIELD } from './queries'; @@ -89,7 +90,14 @@ export async function uploadFile(options: UploadFileOptions): Promise void, signal?: AbortSignal, ): Promise { // Use XMLHttpRequest for progress support if available if (typeof XMLHttpRequest !== 'undefined' && onProgress) { - return putWithXHR(url, file, contentType, onProgress, signal); + return putWithXHR(url, file, contentType, contentHash, onProgress, signal); } const body = await file.arrayBuffer(); - await putToPresignedUrl(url, body, contentType, signal); + await putToPresignedUrl(url, body, contentType, { signal, contentHash }); } /** @@ -174,13 +183,16 @@ function putWithXHR( url: string, file: UploadFileOptions['file'], contentType: string, + contentHash: string, onProgress: (percent: number) => void, signal?: AbortSignal, ): Promise { return new Promise((resolve, reject) => { const xhr = new XMLHttpRequest(); xhr.open('PUT', url); - xhr.setRequestHeader('Content-Type', contentType); + for (const [name, value] of Object.entries(presignedPutHeaders(contentType, contentHash))) { + xhr.setRequestHeader(name, value); + } xhr.upload.addEventListener('progress', (event) => { if (event.lengthComputable) {