From 7a879d3840bb1cbea7829652678fd31ae72c6ead Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Wed, 11 Nov 2026 02:17:59 +0000 Subject: [PATCH 1/2] feat(uploads): bind presigned PUT URLs to the file row (x-amz-meta-upload-id) and return uploadHeaders --- .../__tests__/s3-signer.integration.test.ts | 46 +++++++++++++++++ .../src/index.ts | 3 ++ .../src/plugin.ts | 19 ++++++- .../src/s3-signer.ts | 49 +++++++++++++++++-- .../__snapshots__/storage.node.test.ts.snap | 1 + graphql/query/__tests__/storage.node.test.ts | 31 +++++++++++- graphql/query/src/storage/document.ts | 3 +- graphql/query/src/storage/transport.ts | 5 +- graphql/query/src/storage/upload.ts | 11 ++++- .../bucket-provisioner/__tests__/cors.test.ts | 2 + packages/bucket-provisioner/src/cors.ts | 2 + .../upload-client/__tests__/upload.test.ts | 28 +++++++++++ packages/upload-client/src/put.ts | 15 ++++-- packages/upload-client/src/queries.ts | 1 + .../upload-client/src/storage-transport.ts | 6 ++- packages/upload-client/src/types.ts | 6 +++ packages/upload-client/src/upload.ts | 16 +++--- 17 files changed, 219 insertions(+), 25 deletions(-) diff --git a/graphile/graphile-presigned-url-plugin/__tests__/s3-signer.integration.test.ts b/graphile/graphile-presigned-url-plugin/__tests__/s3-signer.integration.test.ts index 972c989bbd..15ce81b7ff 100644 --- a/graphile/graphile-presigned-url-plugin/__tests__/s3-signer.integration.test.ts +++ b/graphile/graphile-presigned-url-plugin/__tests__/s3-signer.integration.test.ts @@ -20,6 +20,8 @@ import { generatePresignedGetUrl, generatePresignedPutUrl, headObject, + presignedPutHeaders, + UPLOAD_ID_HEADER, } from '../src/s3-signer'; import type { S3Config } from '../src/types'; @@ -216,6 +218,50 @@ describe('s3-signer integration (RustFS)', () => { }); }); + describe('row-bound presigned PUT (x-amz-meta-upload-id names the file row)', () => { + const content = 'bytes this upload writes'; + const hash = sha256Hex(content); + const fileId = '11111111-1111-4111-8111-111111111111'; + const otherFileId = '22222222-2222-4222-8222-222222222222'; + + async function boundUrl(key: string): Promise { + return generatePresignedPutUrl(s3Config, key, 'text/plain', Buffer.byteLength(content), 300, hash, fileId); + } + + it('signs the upload id as a header, not a query parameter', async () => { + const url = new URL(await boundUrl(`row-bound/${Date.now()}-q`)); + expect(url.searchParams.get('X-Amz-SignedHeaders')).toContain(UPLOAD_ID_HEADER); + expect(url.searchParams.has(UPLOAD_ID_HEADER)).toBe(false); + }); + + it('accepts the PUT that sends the headers it was issued with, and HEAD names the row', async () => { + const key = `row-bound/${Date.now()}-ok`; + const headers = presignedPutHeaders('text/plain', hash, fileId); + const res = await fetch(await boundUrl(key), { method: 'PUT', headers, body: content }); + expect(res.status).toBe(200); + expect(await headObject(s3Config, key)).toEqual(expect.objectContaining({ uploadId: fileId })); + }); + + it('refuses a PUT that omits the upload id header (signature covers it)', async () => { + const key = `row-bound/${Date.now()}-missing`; + const res = await uploadToPresignedUrl(await boundUrl(key), content, 'text/plain', { + [CHECKSUM_SHA256_HEADER]: checksumSha256Base64(hash), + }); + expect(res.status).toBe(403); + expect(await headObject(s3Config, key)).toBeNull(); + }); + + it("refuses a PUT that names another row's id", async () => { + const key = `row-bound/${Date.now()}-other`; + const res = await uploadToPresignedUrl(await boundUrl(key), content, 'text/plain', { + [CHECKSUM_SHA256_HEADER]: checksumSha256Base64(hash), + [UPLOAD_ID_HEADER]: otherFileId, + }); + expect(res.status).toBe(403); + expect(await headObject(s3Config, key)).toBeNull(); + }); + }); + describe('headObject', () => { const HEAD_KEY = 'test-head-object.json'; const HEAD_CONTENT = JSON.stringify({ test: true }); diff --git a/graphile/graphile-presigned-url-plugin/src/index.ts b/graphile/graphile-presigned-url-plugin/src/index.ts index 7852a54f21..59c6c71554 100644 --- a/graphile/graphile-presigned-url-plugin/src/index.ts +++ b/graphile/graphile-presigned-url-plugin/src/index.ts @@ -46,6 +46,9 @@ export { generatePresignedGetUrl, generatePresignedPutUrl, headObject, + presignedPutHeaders, + UPLOAD_ID_HEADER, + UPLOAD_ID_METADATA_KEY, } from './s3-signer'; export { clearBucketCache, clearStorageModuleCache, getBucketConfig, loadAllStorageModules,resolveStorageConfigFromCodec, resolveStorageModuleByFileId } from './storage-module-cache'; export type { diff --git a/graphile/graphile-presigned-url-plugin/src/plugin.ts b/graphile/graphile-presigned-url-plugin/src/plugin.ts index 2d1754372f..298ee6371d 100644 --- a/graphile/graphile-presigned-url-plugin/src/plugin.ts +++ b/graphile/graphile-presigned-url-plugin/src/plugin.ts @@ -34,7 +34,7 @@ import { isLiveFileRow } from './file-lifecycle'; import { fileNotReplaceable } from './file-not-replaceable'; import { buildFileProjection, type FileProjection } from './file-projection'; import { assertBucketReconciled, resolveS3ForDatabase } from './physical-bucket'; -import { deleteS3Object,generatePresignedPutUrl } from './s3-signer'; +import { deleteS3Object,generatePresignedPutUrl, presignedPutHeaders } from './s3-signer'; import { recordManagedFile } from './storage-file-recorder'; import { getBucketConfig, loadAllStorageModules, readBucketPhysicalName, resolveStorageConfigFromCodec } from './storage-module-cache'; import type { BucketConfig,PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types'; @@ -212,6 +212,17 @@ export function createPresignedUrlPlugin( name: names.uploadPayloadType, fields: { uploadUrl: { type: GraphQLString, description: 'Presigned PUT URL (null if deduplicated)' }, + ...(jsonType + ? { + uploadHeaders: { + type: jsonType, + description: + 'The exact headers the PUT to uploadUrl must send (Content-Type and every header the URL is ' + + 'signed over, e.g. x-amz-checksum-sha256 and x-amz-meta-upload-id). Forward them as-is. ' + + 'Null if deduplicated.', + }, + } + : {}), fileId: { type: new GraphQLNonNull(GraphQLString), description: 'The file ID (UUID)' }, key: { type: new GraphQLNonNull(GraphQLString), description: 'The S3 object key' }, deduplicated: { type: new GraphQLNonNull(GraphQLBoolean), description: 'Whether this file was deduplicated (content already exists)' }, @@ -338,6 +349,7 @@ export function createPresignedUrlPlugin( name: names.bulkUploadFilePayloadType, fields: { uploadUrl: { type: GraphQLString }, + ...(jsonType ? { uploadHeaders: { type: jsonType, description: 'The exact headers the PUT to uploadUrl must send.' } } : {}), fileId: { type: new GraphQLNonNull(GraphQLString) }, key: { type: new GraphQLNonNull(GraphQLString) }, deduplicated: { type: new GraphQLNonNull(GraphQLBoolean) }, @@ -741,6 +753,7 @@ export async function processSingleFile( log.info(`Dedup hit (custom key): file ${existing.id} for key ${s3Key}`); return { uploadUrl: null as string | null, + uploadHeaders: null as Record | null, fileId: existing.id as string, key: s3Key, deduplicated: true, @@ -789,6 +802,7 @@ export async function processSingleFile( return { uploadUrl: null as string | null, + uploadHeaders: null as Record | null, fileId: existingFile.id as string, key: existingKey, deduplicated: true, @@ -839,12 +853,15 @@ export async function processSingleFile( size, storageConfig.uploadUrlExpirySeconds, contentHash, + fileId, ); + const uploadHeaders = presignedPutHeaders(contentType, contentHash, fileId); const expiresAt = new Date(Date.now() + storageConfig.uploadUrlExpirySeconds * 1000).toISOString(); return { uploadUrl, + uploadHeaders, fileId, key: s3Key, deduplicated: false, diff --git a/graphile/graphile-presigned-url-plugin/src/s3-signer.ts b/graphile/graphile-presigned-url-plugin/src/s3-signer.ts index 96e13ebb9d..cccc05866e 100644 --- a/graphile/graphile-presigned-url-plugin/src/s3-signer.ts +++ b/graphile/graphile-presigned-url-plugin/src/s3-signer.ts @@ -16,6 +16,17 @@ const log = new Logger('graphile-presigned-url:s3'); /** The header a checksum-bound presigned PUT must carry (RFC 4648 base64 of the raw digest). */ export const CHECKSUM_SHA256_HEADER = 'x-amz-checksum-sha256'; +/** + * The header a row-bound presigned PUT must carry: the id of the file row the + * URL was issued for, stored as the object's `upload-id` metadata. The confirm + * worker accepts only an object whose metadata names its own row, so bytes at + * the key that some other upload wrote are never adopted. + */ +export const UPLOAD_ID_HEADER = 'x-amz-meta-upload-id'; + +/** The metadata key {@link UPLOAD_ID_HEADER} is stored under (and HEAD returns). */ +export const UPLOAD_ID_METADATA_KEY = 'upload-id'; + /** * Convert a hex-encoded SHA-256 (the `content_hash` clients send) to the * base64-of-raw-digest form S3 expects in `x-amz-checksum-sha256`. @@ -27,6 +38,23 @@ export function checksumSha256Base64(contentHashHex: string): string { return Buffer.from(contentHashHex, 'hex').toString('base64'); } +/** + * The exact headers a PUT to {@link generatePresignedPutUrl}'s URL must send, + * for the same arguments. Returned to clients as `uploadHeaders` so they forward + * them rather than rebuild them, which makes the next signed header free. + */ +export function presignedPutHeaders( + contentType: string, + contentHash?: string, + uploadId?: string, +): Record { + return { + 'Content-Type': contentType, + ...(contentHash ? { [CHECKSUM_SHA256_HEADER]: checksumSha256Base64(contentHash) } : {}), + ...(uploadId ? { [UPLOAD_ID_HEADER]: uploadId } : {}), + }; +} + /** * Generate a presigned PUT URL for uploading a file to S3. * @@ -40,12 +68,18 @@ export function checksumSha256Base64(contentHashHex: string): string { * (`BadDigest`), and a PUT without the header fails the signature. This is * what makes a content-addressed key actually hold the content it names. * + * When `uploadId` is given, it is signed in the same way as + * `x-amz-meta-upload-id`, so the stored object names the file row whose upload + * wrote it. A PUT without the header, or with another row's id, fails the + * signature. + * * @param s3Config - S3 client and bucket configuration * @param key - S3 object key (content hash or UUID) * @param contentType - MIME type (locked into the signature) * @param contentLength - File size in bytes (locked into the signature) * @param expiresIn - URL expiry in seconds (default: 900 = 15 minutes) * @param contentHash - Hex SHA-256 the uploaded bytes must hash to + * @param uploadId - Id of the file row the upload is for * @returns Presigned PUT URL */ export async function generatePresignedPutUrl( @@ -55,6 +89,7 @@ export async function generatePresignedPutUrl( contentLength: number, expiresIn: number = 900, contentHash?: string, + uploadId?: string, ): Promise { const command = new PutObjectCommand({ Bucket: s3Config.bucket, @@ -62,13 +97,18 @@ export async function generatePresignedPutUrl( ContentType: contentType, ContentLength: contentLength, ...(contentHash ? { ChecksumSHA256: checksumSha256Base64(contentHash) } : {}), + ...(uploadId ? { Metadata: { [UPLOAD_ID_METADATA_KEY]: uploadId } } : {}), }); + const unhoistableHeaders = new Set([ + ...(contentHash ? [CHECKSUM_SHA256_HEADER] : []), + ...(uploadId ? [UPLOAD_ID_HEADER] : []), + ]); let url: string; try { url = await getSignedUrl(s3Config.client as any, command, { expiresIn, - ...(contentHash ? { unhoistableHeaders: new Set([CHECKSUM_SHA256_HEADER]) } : {}), + ...(unhoistableHeaders.size > 0 ? { unhoistableHeaders } : {}), }); } catch (err) { throw s3FailureError( @@ -78,7 +118,7 @@ export async function generatePresignedPutUrl( ); } log.debug( - `Generated presigned PUT URL for key=${key}, contentType=${contentType}, checksum=${contentHash ? 'sha256' : 'none'}, expires=${expiresIn}s`, + `Generated presigned PUT URL for key=${key}, contentType=${contentType}, checksum=${contentHash ? 'sha256' : 'none'}, uploadId=${uploadId ?? 'none'}, expires=${expiresIn}s`, ); return url; } @@ -190,7 +230,7 @@ export async function headObject( s3Config: S3Config, key: string, expectedContentType?: string, -): Promise<{ contentType?: string; contentLength?: number; checksumSha256?: string } | null> { +): Promise<{ contentType?: string; contentLength?: number; checksumSha256?: string; uploadId?: string } | null> { try { const response = await s3Config.client.send( new HeadObjectCommand({ @@ -210,6 +250,9 @@ export async function headObject( contentType: response.ContentType, contentLength: response.ContentLength, ...(response.ChecksumSHA256 ? { checksumSha256: response.ChecksumSHA256 } : {}), + ...(response.Metadata?.[UPLOAD_ID_METADATA_KEY] + ? { uploadId: response.Metadata[UPLOAD_ID_METADATA_KEY] } + : {}), }; } catch (e: any) { if (e.name === 'NotFound' || e.$metadata?.httpStatusCode === 404) { diff --git a/graphql/query/__tests__/__snapshots__/storage.node.test.ts.snap b/graphql/query/__tests__/__snapshots__/storage.node.test.ts.snap index bd260004d7..4cc117387c 100644 --- a/graphql/query/__tests__/__snapshots__/storage.node.test.ts.snap +++ b/graphql/query/__tests__/__snapshots__/storage.node.test.ts.snap @@ -13,6 +13,7 @@ exports[`buildUploadDocument() builds the mutation entirely from the surface 1`] "mutation UploadFileMutation($input: UploadFileInput!) { uploadFile(input: $input) { uploadUrl + uploadHeaders fileId key deduplicated diff --git a/graphql/query/__tests__/storage.node.test.ts b/graphql/query/__tests__/storage.node.test.ts index 8dc92bd7dc..442f7b6daa 100644 --- a/graphql/query/__tests__/storage.node.test.ts +++ b/graphql/query/__tests__/storage.node.test.ts @@ -110,14 +110,15 @@ interface PutCall { url: string; contentType: string; size: number; + headers?: Record; } function fakeTransport(puts: PutCall[] = []): StorageTransport { return { hashFile: async (file) => createHash('sha256').update(Buffer.from(await file.arrayBuffer())).digest('hex'), - putObject: async (url, body, contentType) => { - puts.push({ url, contentType, size: body.byteLength }); + putObject: async (url, body, contentType, _signal, _contentHash, headers) => { + puts.push({ url, contentType, size: body.byteLength, ...(headers ? { headers } : {}) }); }, }; } @@ -207,6 +208,7 @@ describe('buildUploadDocument()', () => { expect(document).toContain('mutation UploadFileMutation($input: UploadFileInput!)'); expect(document).toContain('uploadFile(input: $input)'); expect(document).toContain('uploadUrl'); + expect(document).toContain('uploadHeaders'); expect(document).toContain('previousVersionId'); }); }); @@ -355,6 +357,31 @@ describe('createStorageClient()', () => { ]); }); + it('hands the transport the exact uploadHeaders the server returned', async () => { + const uploadHeaders = { + 'Content-Type': 'text/plain', + 'x-amz-checksum-sha256': 'base64digest=', + 'x-amz-meta-upload-id': 'file-3', + }; + const { client, puts } = clientWith({ + uploadFile: { + uploadUrl: 'https://s3.example/presigned', + uploadHeaders, + fileId: 'file-3', + key: 'ghi789', + deduplicated: false, + expiresAt: '2030-01-01T00:00:00Z', + previousVersionId: null, + }, + }); + + await client.upload({ filesTable: 'files' }, { file: fakeFile() }); + + expect(puts).toEqual([ + { url: 'https://s3.example/presigned', contentType: 'text/plain', size: 11, headers: uploadHeaders }, + ]); + }); + it('refuses to upload without a transport adapter', async () => { const execute: GraphQLExecutor = async () => metaResult() as unknown as Record; diff --git a/graphql/query/src/storage/document.ts b/graphql/query/src/storage/document.ts index f00239eb5d..c0dc505b62 100644 --- a/graphql/query/src/storage/document.ts +++ b/graphql/query/src/storage/document.ts @@ -12,6 +12,7 @@ import type { StorageSurface } from './types'; const UPLOAD_PAYLOAD_FIELDS = [ 'uploadUrl', + 'uploadHeaders', 'fileId', 'key', 'deduplicated', @@ -24,7 +25,7 @@ const UPLOAD_PAYLOAD_FIELDS = [ * * mutation UploadAppFileMutation($input: UploadAppFileInput!) { * uploadAppFile(input: $input) { - * uploadUrl fileId key deduplicated expiresAt previousVersionId + * uploadUrl uploadHeaders fileId key deduplicated expiresAt previousVersionId * } * } */ diff --git a/graphql/query/src/storage/transport.ts b/graphql/query/src/storage/transport.ts index 11b7a7c004..672a9a2140 100644 --- a/graphql/query/src/storage/transport.ts +++ b/graphql/query/src/storage/transport.ts @@ -26,7 +26,9 @@ export interface StorageTransport { /** * PUT the bytes to a presigned URL. `contentHash` is the hex SHA-256 the * upload was requested with; the URL is signed over it as - * `x-amz-checksum-sha256`, so the transport must send it. + * `x-amz-checksum-sha256`, so the transport must send it. `headers` are the + * payload's `uploadHeaders` — the exact headers the URL needs (checksum and + * `x-amz-meta-upload-id` included); a transport sends them as-is when given. */ putObject( url: string, @@ -34,6 +36,7 @@ export interface StorageTransport { contentType: string, signal?: AbortSignal, contentHash?: string, + headers?: Record, ): Promise; } diff --git a/graphql/query/src/storage/upload.ts b/graphql/query/src/storage/upload.ts index f94077288f..7beaf92b44 100644 --- a/graphql/query/src/storage/upload.ts +++ b/graphql/query/src/storage/upload.ts @@ -94,7 +94,7 @@ export async function uploadToSurface( } const payload = data?.[surface.upload.mutation] as - | (StorageUploadResult & { uploadUrl: string | null }) + | (StorageUploadResult & { uploadUrl: string | null; uploadHeaders?: Record | null }) | undefined; if (!payload) { throw new StorageError( @@ -114,7 +114,14 @@ export async function uploadToSurface( } checkAborted(signal); - await transport.putObject(payload.uploadUrl, await file.arrayBuffer(), contentType, signal, contentHash); + await transport.putObject( + payload.uploadUrl, + await file.arrayBuffer(), + contentType, + signal, + contentHash, + payload.uploadHeaders ?? undefined, + ); return toResult(payload); } diff --git a/packages/bucket-provisioner/__tests__/cors.test.ts b/packages/bucket-provisioner/__tests__/cors.test.ts index 9c6dcaabab..1e11fa22d0 100644 --- a/packages/bucket-provisioner/__tests__/cors.test.ts +++ b/packages/bucket-provisioner/__tests__/cors.test.ts @@ -24,6 +24,7 @@ describe('buildUploadCorsRules', () => { expect(rule.allowedHeaders).toContain('Content-Length'); expect(rule.allowedHeaders).toContain('Authorization'); expect(rule.allowedHeaders).toContain('x-amz-checksum-sha256'); + expect(rule.allowedHeaders).toContain('x-amz-meta-upload-id'); }); it('exposes ETag and Content-Length', () => { @@ -76,6 +77,7 @@ describe('buildPrivateCorsRules', () => { expect(rule.allowedHeaders).toContain('Content-Length'); expect(rule.allowedHeaders).toContain('Authorization'); expect(rule.allowedHeaders).toContain('x-amz-checksum-sha256'); + expect(rule.allowedHeaders).toContain('x-amz-meta-upload-id'); }); it('uses default maxAgeSeconds of 3600', () => { diff --git a/packages/bucket-provisioner/src/cors.ts b/packages/bucket-provisioner/src/cors.ts index d3a632854b..c5e9c6ea08 100644 --- a/packages/bucket-provisioner/src/cors.ts +++ b/packages/bucket-provisioner/src/cors.ts @@ -39,6 +39,7 @@ export function buildUploadCorsRules( 'Content-MD5', 'x-amz-content-sha256', 'x-amz-checksum-sha256', + 'x-amz-meta-upload-id', 'x-amz-date', 'x-amz-security-token', 'Authorization', @@ -83,6 +84,7 @@ export function buildPrivateCorsRules( 'Content-MD5', 'x-amz-content-sha256', 'x-amz-checksum-sha256', + 'x-amz-meta-upload-id', 'x-amz-date', 'x-amz-security-token', 'Authorization', diff --git a/packages/upload-client/__tests__/upload.test.ts b/packages/upload-client/__tests__/upload.test.ts index 93358d2020..26e630669a 100644 --- a/packages/upload-client/__tests__/upload.test.ts +++ b/packages/upload-client/__tests__/upload.test.ts @@ -115,6 +115,34 @@ describe('uploadFile', () => { }); }); + describe('server-issued upload headers', () => { + it('forwards uploadHeaders as-is instead of rebuilding them', async () => { + const uploadHeaders = { + 'Content-Type': 'text/plain', + 'x-amz-checksum-sha256': 'uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=', + 'x-amz-meta-upload-id': 'file-uuid-456', + }; + const { execute, calls } = createMockExecutor({ + uploadUrl: 'https://s3.example.com/presigned-put-url', + uploadHeaders, + fileId: 'file-uuid-456', + key: HELLO_WORLD_HASH, + deduplicated: false, + expiresAt: new Date(Date.now() + 900_000).toISOString(), + previousVersionId: null, + }); + global.fetch = jest.fn().mockResolvedValue({ ok: true, status: 200, text: async () => '' }); + + await uploadFile({ file: createMockFile('hello world'), bucketKey: 'avatars', execute }); + + expect(calls[0].query).toContain('uploadHeaders'); + expect(global.fetch).toHaveBeenCalledWith( + 'https://s3.example.com/presigned-put-url', + expect.objectContaining({ method: 'PUT', headers: uploadHeaders }), + ); + }); + }); + describe('deduplicated upload', () => { it('should skip PUT when deduplicated', async () => { const file = createMockFile('hello world'); diff --git a/packages/upload-client/src/put.ts b/packages/upload-client/src/put.ts index 9153cfa228..0cc0dc7827 100644 --- a/packages/upload-client/src/put.ts +++ b/packages/upload-client/src/put.ts @@ -35,6 +35,13 @@ export interface PutToPresignedUrlOptions { * typed array, Blob); a streamed body must pass it explicitly. */ contentHash?: string; + /** + * The exact headers to send, as `requestUploadUrl` returned them in + * `uploadHeaders`. When given they are sent as-is (they already carry + * `Content-Type`, the checksum and the upload id the URL is signed over) and + * `contentHash` is not needed. + */ + headers?: Record; } function normalizePutOptions( @@ -56,7 +63,8 @@ function normalizePutOptions( * @param url - Presigned PUT URL * @param body - Content to upload * @param contentType - MIME type (must match the presigned URL's content-type constraint) - * @param signal - Optional AbortSignal for cancellation + * @param signalOrOptions - AbortSignal, or options: `headers` (the payload's + * `uploadHeaders`), `contentHash`, `signal` * @returns The fetch Response (already verified as ok) * @throws {UploadError} If the PUT fails or is aborted */ @@ -66,12 +74,11 @@ export async function putToPresignedUrl( contentType: string, signalOrOptions?: AbortSignal | PutToPresignedUrlOptions, ): Promise { - const { signal, contentHash } = normalizePutOptions(signalOrOptions); + const { signal, contentHash, headers } = normalizePutOptions(signalOrOptions); try { - const checksumSource = contentHash ?? (await sha256HexOfBody(body)); const response = await fetch(url, { method: 'PUT', - headers: presignedPutHeaders(contentType, checksumSource), + headers: headers ?? presignedPutHeaders(contentType, contentHash ?? (await sha256HexOfBody(body))), body: body as BodyInit, signal, }); diff --git a/packages/upload-client/src/queries.ts b/packages/upload-client/src/queries.ts index 54262d38d4..8c11b46803 100644 --- a/packages/upload-client/src/queries.ts +++ b/packages/upload-client/src/queries.ts @@ -28,6 +28,7 @@ export function buildRequestUploadUrlQuery(bucketQueryField: string): string { filename: $filename ) { uploadUrl + uploadHeaders fileId key deduplicated diff --git a/packages/upload-client/src/storage-transport.ts b/packages/upload-client/src/storage-transport.ts index 535541f54c..f4e34f96e1 100644 --- a/packages/upload-client/src/storage-transport.ts +++ b/packages/upload-client/src/storage-transport.ts @@ -16,6 +16,7 @@ export interface StorageTransportPort { /** * PUT the bytes to a presigned URL. `contentHash` is the hex SHA-256 the * URL was requested with; it is sent as the signed `x-amz-checksum-sha256`. + * `headers` are the payload's `uploadHeaders`; when given they are sent as-is. */ putObject( url: string, @@ -23,14 +24,15 @@ export interface StorageTransportPort { contentType: string, signal?: AbortSignal, contentHash?: string, + headers?: Record, ): Promise; } /** The presigned-URL transport: SHA-256 hashing plus a checksum-bound presigned PUT. */ export const storageTransport: StorageTransportPort = { hashFile: (file) => hashFile(file), - putObject: async (url, body, contentType, signal, contentHash) => { - await putToPresignedUrl(url, body, contentType, { signal, contentHash }); + putObject: async (url, body, contentType, signal, contentHash, headers) => { + await putToPresignedUrl(url, body, contentType, { signal, contentHash, headers }); }, }; diff --git a/packages/upload-client/src/types.ts b/packages/upload-client/src/types.ts index ac0b3094b2..02ec547217 100644 --- a/packages/upload-client/src/types.ts +++ b/packages/upload-client/src/types.ts @@ -24,6 +24,12 @@ export interface RequestUploadUrlInput { export interface RequestUploadUrlPayload { /** Presigned PUT URL (null if deduplicated) */ uploadUrl: string | null; + /** + * The exact headers the PUT to `uploadUrl` must send: `Content-Type` and every + * header the URL is signed over (`x-amz-checksum-sha256`, + * `x-amz-meta-upload-id`). Forward them as-is. Null if deduplicated. + */ + uploadHeaders?: Record | null; /** The file ID (UUID) */ fileId: string; /** The S3 object key */ diff --git a/packages/upload-client/src/upload.ts b/packages/upload-client/src/upload.ts index 11dd1db5f6..84a367558c 100644 --- a/packages/upload-client/src/upload.ts +++ b/packages/upload-client/src/upload.ts @@ -90,11 +90,11 @@ export async function uploadFile(options: UploadFileOptions): Promise, onProgress?: (percent: number) => void, signal?: AbortSignal, ): Promise { // Use XMLHttpRequest for progress support if available if (typeof XMLHttpRequest !== 'undefined' && onProgress) { - return putWithXHR(url, file, contentType, contentHash, onProgress, signal); + return putWithXHR(url, file, headers, onProgress, signal); } const body = await file.arrayBuffer(); - await putToPresignedUrl(url, body, contentType, { signal, contentHash }); + await putToPresignedUrl(url, body, headers['Content-Type'], { signal, headers }); } /** @@ -182,15 +181,14 @@ async function putToS3( function putWithXHR( url: string, file: UploadFileOptions['file'], - contentType: string, - contentHash: string, + headers: Record, onProgress: (percent: number) => void, signal?: AbortSignal, ): Promise { return new Promise((resolve, reject) => { const xhr = new XMLHttpRequest(); xhr.open('PUT', url); - for (const [name, value] of Object.entries(presignedPutHeaders(contentType, contentHash))) { + for (const [name, value] of Object.entries(headers)) { xhr.setRequestHeader(name, value); } From 3e894cfba0b70aee3f1df03da14d5b26fc6558fc Mon Sep 17 00:00:00 2001 From: Dan Lynch Date: Wed, 11 Nov 2026 02:29:03 +0000 Subject: [PATCH 2/2] test(server-test): forward the issued uploadHeaders on presigned PUTs --- .../db-scope-upload.integration.test.ts | 5 +++- .../upload-columns.integration.test.ts | 5 ++-- .../__tests__/upload.integration.test.ts | 23 +++++++++++++++---- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/graphql/server-test/__tests__/db-scope-upload.integration.test.ts b/graphql/server-test/__tests__/db-scope-upload.integration.test.ts index 69cd29efc0..80996560e0 100644 --- a/graphql/server-test/__tests__/db-scope-upload.integration.test.ts +++ b/graphql/server-test/__tests__/db-scope-upload.integration.test.ts @@ -69,6 +69,7 @@ const UPLOAD_FILE = ` mutation UploadFile($input: UploadFileInput!) { uploadFile(input: $input) { uploadUrl + uploadHeaders fileId key deduplicated @@ -169,6 +170,7 @@ describe('database-scope upload surface', () => { const contentType = 'text/plain'; let contentHash: string; let uploadUrl: string; + let uploadHeaders: Record; // RustFS uses path-style URLs: http://host:9000//?... const bucketFromPresignedUrl = (url: string): string => @@ -201,10 +203,11 @@ describe('database-scope upload surface', () => { expect(payload.deduplicated).toBe(false); uploadUrl = payload.uploadUrl; + uploadHeaders = payload.uploadHeaders; }); it('accepts the PUT, into a physical bucket recorded on the tenant row', async () => { - const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { contentHash }); + const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { headers: uploadHeaders }); expect(putRes.ok).toBe(true); const stored = await pg.query( diff --git a/graphql/server-test/__tests__/upload-columns.integration.test.ts b/graphql/server-test/__tests__/upload-columns.integration.test.ts index 11968b5f5f..0a0ccf1a9b 100644 --- a/graphql/server-test/__tests__/upload-columns.integration.test.ts +++ b/graphql/server-test/__tests__/upload-columns.integration.test.ts @@ -53,6 +53,7 @@ const UPLOAD_APP_FILE = ` fileId key uploadUrl + uploadHeaders } } `; @@ -151,8 +152,8 @@ describe('image / upload / attachment columns', () => { } }); expect(uploaded.body.errors).toBeUndefined(); - const { fileId, key, uploadUrl } = uploaded.body.data.uploadAppFile; - expect((await putToPresignedUrl(uploadUrl, content, 'text/plain', { contentHash })).ok).toBe(true); + const { fileId, key, uploadUrl, uploadHeaders } = uploaded.body.data.uploadAppFile; + expect((await putToPresignedUrl(uploadUrl, content, 'text/plain', { headers: uploadHeaders })).ok).toBe(true); // The confirm worker is not part of this fixture; it is what moves a row // from `requested` to `uploaded` once it has inspected the object. await pg.query(`UPDATE "${storageSchema}".app_files SET status = 'uploaded' WHERE id = $1`, [fileId]); diff --git a/graphql/server-test/__tests__/upload.integration.test.ts b/graphql/server-test/__tests__/upload.integration.test.ts index 3f86308ae2..78f5ad2e5a 100644 --- a/graphql/server-test/__tests__/upload.integration.test.ts +++ b/graphql/server-test/__tests__/upload.integration.test.ts @@ -90,6 +90,7 @@ const UPLOAD_APP_FILE = ` mutation UploadAppFile($input: UploadAppFileInput!) { uploadAppFile(input: $input) { uploadUrl + uploadHeaders fileId key deduplicated @@ -392,6 +393,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { const contentType = 'text/plain'; let contentHash: string; let uploadUrl: string; + let uploadHeaders: Record; beforeAll(async () => { contentHash = await hashContent(fileContent); @@ -422,10 +424,16 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { expect(payload.expiresAt).toBeTruthy(); uploadUrl = payload.uploadUrl; + uploadHeaders = payload.uploadHeaders; + expect(uploadHeaders).toEqual({ + 'Content-Type': contentType, + 'x-amz-checksum-sha256': Buffer.from(contentHash, 'hex').toString('base64'), + 'x-amz-meta-upload-id': payload.fileId, + }); }); it('should accept a PUT to the presigned URL', async () => { - const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { contentHash }); + const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { headers: uploadHeaders }); expect(putRes.ok).toBe(true); await confirmUploaded(contentHash); }); @@ -436,6 +444,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { const contentType = 'text/plain'; let contentHash: string; let uploadUrl: string; + let uploadHeaders: Record; beforeAll(async () => { contentHash = await hashContent(fileContent); @@ -465,10 +474,16 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { expect(payload.expiresAt).toBeTruthy(); uploadUrl = payload.uploadUrl; + uploadHeaders = payload.uploadHeaders; + expect(uploadHeaders).toEqual({ + 'Content-Type': contentType, + 'x-amz-checksum-sha256': Buffer.from(contentHash, 'hex').toString('base64'), + 'x-amz-meta-upload-id': payload.fileId, + }); }); it('should accept a PUT to the presigned URL', async () => { - const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { contentHash }); + const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { headers: uploadHeaders }); expect(putRes.ok).toBe(true); }); }); @@ -524,7 +539,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { expect(retry.uploadUrl).toBeTruthy(); expect(retry.fileId).not.toBe(first.fileId); - const putRes = await putToPresignedUrl(retry.uploadUrl, fileContent, 'text/plain', { contentHash }); + const putRes = await putToPresignedUrl(retry.uploadUrl, fileContent, 'text/plain', { headers: retry.uploadHeaders }); expect(putRes.ok).toBe(true); await confirmUploaded(contentHash); @@ -869,7 +884,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => { expect(payload.fileId).toBeTruthy(); expect(payload.uploadUrl).toBeTruthy(); - const putRes = await putToPresignedUrl(payload.uploadUrl, fileContent, 'text/plain', { contentHash }); + const putRes = await putToPresignedUrl(payload.uploadUrl, fileContent, 'text/plain', { headers: payload.uploadHeaders }); expect(putRes.ok).toBe(true); });