Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import {
generatePresignedGetUrl,
generatePresignedPutUrl,
headObject,
presignedPutHeaders,
UPLOAD_ID_HEADER,
} from '../src/s3-signer';
import type { S3Config } from '../src/types';

Expand Down Expand Up @@ -216,6 +218,50 @@ describe('s3-signer integration (RustFS)', () => {
});
});

describe('row-bound presigned PUT (x-amz-meta-upload-id names the file row)', () => {
const content = 'bytes this upload writes';
const hash = sha256Hex(content);
const fileId = '11111111-1111-4111-8111-111111111111';
const otherFileId = '22222222-2222-4222-8222-222222222222';

async function boundUrl(key: string): Promise<string> {
return generatePresignedPutUrl(s3Config, key, 'text/plain', Buffer.byteLength(content), 300, hash, fileId);
}

it('signs the upload id as a header, not a query parameter', async () => {
const url = new URL(await boundUrl(`row-bound/${Date.now()}-q`));
expect(url.searchParams.get('X-Amz-SignedHeaders')).toContain(UPLOAD_ID_HEADER);
expect(url.searchParams.has(UPLOAD_ID_HEADER)).toBe(false);
});

it('accepts the PUT that sends the headers it was issued with, and HEAD names the row', async () => {
const key = `row-bound/${Date.now()}-ok`;
const headers = presignedPutHeaders('text/plain', hash, fileId);
const res = await fetch(await boundUrl(key), { method: 'PUT', headers, body: content });
expect(res.status).toBe(200);
expect(await headObject(s3Config, key)).toEqual(expect.objectContaining({ uploadId: fileId }));
});

it('refuses a PUT that omits the upload id header (signature covers it)', async () => {
const key = `row-bound/${Date.now()}-missing`;
const res = await uploadToPresignedUrl(await boundUrl(key), content, 'text/plain', {
[CHECKSUM_SHA256_HEADER]: checksumSha256Base64(hash),
});
expect(res.status).toBe(403);
expect(await headObject(s3Config, key)).toBeNull();
});

it("refuses a PUT that names another row's id", async () => {
const key = `row-bound/${Date.now()}-other`;
const res = await uploadToPresignedUrl(await boundUrl(key), content, 'text/plain', {
[CHECKSUM_SHA256_HEADER]: checksumSha256Base64(hash),
[UPLOAD_ID_HEADER]: otherFileId,
});
expect(res.status).toBe(403);
expect(await headObject(s3Config, key)).toBeNull();
});
});

describe('headObject', () => {
const HEAD_KEY = 'test-head-object.json';
const HEAD_CONTENT = JSON.stringify({ test: true });
Expand Down
3 changes: 3 additions & 0 deletions graphile/graphile-presigned-url-plugin/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ export {
generatePresignedGetUrl,
generatePresignedPutUrl,
headObject,
presignedPutHeaders,
UPLOAD_ID_HEADER,
UPLOAD_ID_METADATA_KEY,
} from './s3-signer';
export { clearBucketCache, clearStorageModuleCache, getBucketConfig, loadAllStorageModules,resolveStorageConfigFromCodec, resolveStorageModuleByFileId } from './storage-module-cache';
export type {
Expand Down
19 changes: 18 additions & 1 deletion graphile/graphile-presigned-url-plugin/src/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import { isLiveFileRow } from './file-lifecycle';
import { fileNotReplaceable } from './file-not-replaceable';
import { buildFileProjection, type FileProjection } from './file-projection';
import { assertBucketReconciled, resolveS3ForDatabase } from './physical-bucket';
import { deleteS3Object,generatePresignedPutUrl } from './s3-signer';
import { deleteS3Object,generatePresignedPutUrl, presignedPutHeaders } from './s3-signer';
import { recordManagedFile } from './storage-file-recorder';
import { getBucketConfig, loadAllStorageModules, readBucketPhysicalName, resolveStorageConfigFromCodec } from './storage-module-cache';
import type { BucketConfig,PresignedUrlPluginOptions, S3Config, StorageModuleConfig } from './types';
Expand Down Expand Up @@ -212,6 +212,17 @@ export function createPresignedUrlPlugin(
name: names.uploadPayloadType,
fields: {
uploadUrl: { type: GraphQLString, description: 'Presigned PUT URL (null if deduplicated)' },
...(jsonType
? {
uploadHeaders: {
type: jsonType,
description:
'The exact headers the PUT to uploadUrl must send (Content-Type and every header the URL is ' +
'signed over, e.g. x-amz-checksum-sha256 and x-amz-meta-upload-id). Forward them as-is. ' +
'Null if deduplicated.',
},
}
: {}),
fileId: { type: new GraphQLNonNull(GraphQLString), description: 'The file ID (UUID)' },
key: { type: new GraphQLNonNull(GraphQLString), description: 'The S3 object key' },
deduplicated: { type: new GraphQLNonNull(GraphQLBoolean), description: 'Whether this file was deduplicated (content already exists)' },
Expand Down Expand Up @@ -338,6 +349,7 @@ export function createPresignedUrlPlugin(
name: names.bulkUploadFilePayloadType,
fields: {
uploadUrl: { type: GraphQLString },
...(jsonType ? { uploadHeaders: { type: jsonType, description: 'The exact headers the PUT to uploadUrl must send.' } } : {}),
fileId: { type: new GraphQLNonNull(GraphQLString) },
key: { type: new GraphQLNonNull(GraphQLString) },
deduplicated: { type: new GraphQLNonNull(GraphQLBoolean) },
Expand Down Expand Up @@ -741,6 +753,7 @@ export async function processSingleFile(
log.info(`Dedup hit (custom key): file ${existing.id} for key ${s3Key}`);
return {
uploadUrl: null as string | null,
uploadHeaders: null as Record<string, string> | null,
fileId: existing.id as string,
key: s3Key,
deduplicated: true,
Expand Down Expand Up @@ -789,6 +802,7 @@ export async function processSingleFile(

return {
uploadUrl: null as string | null,
uploadHeaders: null as Record<string, string> | null,
fileId: existingFile.id as string,
key: existingKey,
deduplicated: true,
Expand Down Expand Up @@ -839,12 +853,15 @@ export async function processSingleFile(
size,
storageConfig.uploadUrlExpirySeconds,
contentHash,
fileId,
);
const uploadHeaders = presignedPutHeaders(contentType, contentHash, fileId);

const expiresAt = new Date(Date.now() + storageConfig.uploadUrlExpirySeconds * 1000).toISOString();

return {
uploadUrl,
uploadHeaders,
fileId,
key: s3Key,
deduplicated: false,
Expand Down
49 changes: 46 additions & 3 deletions graphile/graphile-presigned-url-plugin/src/s3-signer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,17 @@ const log = new Logger('graphile-presigned-url:s3');
/** The header a checksum-bound presigned PUT must carry (RFC 4648 base64 of the raw digest). */
export const CHECKSUM_SHA256_HEADER = 'x-amz-checksum-sha256';

/**
* The header a row-bound presigned PUT must carry: the id of the file row the
* URL was issued for, stored as the object's `upload-id` metadata. The confirm
* worker accepts only an object whose metadata names its own row, so bytes at
* the key that some other upload wrote are never adopted.
*/
export const UPLOAD_ID_HEADER = 'x-amz-meta-upload-id';

/** The metadata key {@link UPLOAD_ID_HEADER} is stored under (and HEAD returns). */
export const UPLOAD_ID_METADATA_KEY = 'upload-id';

/**
* Convert a hex-encoded SHA-256 (the `content_hash` clients send) to the
* base64-of-raw-digest form S3 expects in `x-amz-checksum-sha256`.
Expand All @@ -27,6 +38,23 @@ export function checksumSha256Base64(contentHashHex: string): string {
return Buffer.from(contentHashHex, 'hex').toString('base64');
}

/**
* The exact headers a PUT to {@link generatePresignedPutUrl}'s URL must send,
* for the same arguments. Returned to clients as `uploadHeaders` so they forward
* them rather than rebuild them, which makes the next signed header free.
*/
export function presignedPutHeaders(
contentType: string,
contentHash?: string,
uploadId?: string,
): Record<string, string> {
return {
'Content-Type': contentType,
...(contentHash ? { [CHECKSUM_SHA256_HEADER]: checksumSha256Base64(contentHash) } : {}),
...(uploadId ? { [UPLOAD_ID_HEADER]: uploadId } : {}),
};
}

/**
* Generate a presigned PUT URL for uploading a file to S3.
*
Expand All @@ -40,12 +68,18 @@ export function checksumSha256Base64(contentHashHex: string): string {
* (`BadDigest`), and a PUT without the header fails the signature. This is
* what makes a content-addressed key actually hold the content it names.
*
* When `uploadId` is given, it is signed in the same way as
* `x-amz-meta-upload-id`, so the stored object names the file row whose upload
* wrote it. A PUT without the header, or with another row's id, fails the
* signature.
*
* @param s3Config - S3 client and bucket configuration
* @param key - S3 object key (content hash or UUID)
* @param contentType - MIME type (locked into the signature)
* @param contentLength - File size in bytes (locked into the signature)
* @param expiresIn - URL expiry in seconds (default: 900 = 15 minutes)
* @param contentHash - Hex SHA-256 the uploaded bytes must hash to
* @param uploadId - Id of the file row the upload is for
* @returns Presigned PUT URL
*/
export async function generatePresignedPutUrl(
Expand All @@ -55,20 +89,26 @@ export async function generatePresignedPutUrl(
contentLength: number,
expiresIn: number = 900,
contentHash?: string,
uploadId?: string,
): Promise<string> {
const command = new PutObjectCommand({
Bucket: s3Config.bucket,
Key: key,
ContentType: contentType,
ContentLength: contentLength,
...(contentHash ? { ChecksumSHA256: checksumSha256Base64(contentHash) } : {}),
...(uploadId ? { Metadata: { [UPLOAD_ID_METADATA_KEY]: uploadId } } : {}),
});
const unhoistableHeaders = new Set([
...(contentHash ? [CHECKSUM_SHA256_HEADER] : []),
...(uploadId ? [UPLOAD_ID_HEADER] : []),

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 compatibility · medium

Presigned PUT now signed over x-amz-meta-upload-id, breaking old clients

The server now binds the presigned PUT signature to x-amz-meta-upload-id: presignedPutHeaders adds the header when uploadId is given and generatePresignedPutUrl marks it as unhoistable so S3 requires it to match the signature (graphile/graphile-presigned-url-plugin/src/plugin.ts:858). Clients built against the previous release — including published versions of @constructive-io/upload-client and any hand-rolled uploader — send only Content-Type and x-amz-checksum-sha256, so their PUTs now fail with SignatureDoesNotMatch. There is no opt-out or version negotiation, so every upload from an un-upgraded client breaks until it is rebuilt against the new contract.

📋 Prompt for AI Agents

In graphile/graphile-presigned-url-plugin/src/s3-signer.ts (unhoistableHeaders construction, around lines 92-111) and graphile/graphile-presigned-url-plugin/src/plugin.ts (the generatePresignedPutUrl call passing fileId, around line 858), make the x-amz-meta-upload-id header binding opt-in: add a flag (e.g. bindUploadIdHeader) to the plugin options/storage config that defaults to false, thread it through presignedPutHeaders and generatePresignedPutUrl, and only include the header/metadata when it is enabled — otherwise clients built against the previous signing contract fail with SignatureDoesNotMatch. Keep the confirm-worker path tolerant of uploads without the metadata when binding is disabled.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is expected, and it is what the change is for: a PUT that is not bound to its row is refused. Our clients already send the header: dashboard#322 and constructive-desktop#272 are merged, and their PUTs carry x-amz-meta-upload-id: <fileId>. A URL that is not signed over the header accepts it, so they shipped ahead of this PR.

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The diff does not show the fix described in the original finding. The full s3-signer.ts and plugin.ts diffs confirm that uploadId (the fileId) is unconditionally passed to generatePresignedPutUrl and signed into the URL as x-amz-meta-upload-id with no opt-in flag — it is always on.

The user's claim that dashboard#322 and constructive-desktop#272 are already merged and shipping the header may well be true, but it does not address the compatibility risk for any other client (hand-rolled uploaders, third-party integrations, older mobile builds still in the field) that was not updated before this PR merges. The original finding is about the absence of an opt-out mechanism, not about whether specific known clients were updated.

The recommended fix — a bindUploadIdHeader flag defaulting to false so the new signing contract is opt-in — is not present anywhere in the diff. Until that flag (or equivalent version negotiation) is added, the finding stands.

]);

let url: string;
try {
url = await getSignedUrl(s3Config.client as any, command, {
expiresIn,
...(contentHash ? { unhoistableHeaders: new Set([CHECKSUM_SHA256_HEADER]) } : {}),
...(unhoistableHeaders.size > 0 ? { unhoistableHeaders } : {}),
});
} catch (err) {
throw s3FailureError(
Expand All @@ -78,7 +118,7 @@ export async function generatePresignedPutUrl(
);
}
log.debug(
`Generated presigned PUT URL for key=${key}, contentType=${contentType}, checksum=${contentHash ? 'sha256' : 'none'}, expires=${expiresIn}s`,
`Generated presigned PUT URL for key=${key}, contentType=${contentType}, checksum=${contentHash ? 'sha256' : 'none'}, uploadId=${uploadId ?? 'none'}, expires=${expiresIn}s`,
);
return url;
}
Expand Down Expand Up @@ -190,7 +230,7 @@ export async function headObject(
s3Config: S3Config,
key: string,
expectedContentType?: string,
): Promise<{ contentType?: string; contentLength?: number; checksumSha256?: string } | null> {
): Promise<{ contentType?: string; contentLength?: number; checksumSha256?: string; uploadId?: string } | null> {
try {
const response = await s3Config.client.send(
new HeadObjectCommand({
Expand All @@ -210,6 +250,9 @@ export async function headObject(
contentType: response.ContentType,
contentLength: response.ContentLength,
...(response.ChecksumSHA256 ? { checksumSha256: response.ChecksumSHA256 } : {}),
...(response.Metadata?.[UPLOAD_ID_METADATA_KEY]
? { uploadId: response.Metadata[UPLOAD_ID_METADATA_KEY] }
: {}),
};
} catch (e: any) {
if (e.name === 'NotFound' || e.$metadata?.httpStatusCode === 404) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ exports[`buildUploadDocument() builds the mutation entirely from the surface 1`]
"mutation UploadFileMutation($input: UploadFileInput!) {
uploadFile(input: $input) {
uploadUrl
uploadHeaders
fileId
key
deduplicated
Expand Down
31 changes: 29 additions & 2 deletions graphql/query/__tests__/storage.node.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -110,14 +110,15 @@ interface PutCall {
url: string;
contentType: string;
size: number;
headers?: Record<string, string>;
}

function fakeTransport(puts: PutCall[] = []): StorageTransport {
return {
hashFile: async (file) =>
createHash('sha256').update(Buffer.from(await file.arrayBuffer())).digest('hex'),
putObject: async (url, body, contentType) => {
puts.push({ url, contentType, size: body.byteLength });
putObject: async (url, body, contentType, _signal, _contentHash, headers) => {
puts.push({ url, contentType, size: body.byteLength, ...(headers ? { headers } : {}) });
},
};
}
Expand Down Expand Up @@ -207,6 +208,7 @@ describe('buildUploadDocument()', () => {
expect(document).toContain('mutation UploadFileMutation($input: UploadFileInput!)');
expect(document).toContain('uploadFile(input: $input)');
expect(document).toContain('uploadUrl');
expect(document).toContain('uploadHeaders');
expect(document).toContain('previousVersionId');
});
});
Expand Down Expand Up @@ -355,6 +357,31 @@ describe('createStorageClient()', () => {
]);
});

it('hands the transport the exact uploadHeaders the server returned', async () => {
const uploadHeaders = {
'Content-Type': 'text/plain',
'x-amz-checksum-sha256': 'base64digest=',
'x-amz-meta-upload-id': 'file-3',
};
const { client, puts } = clientWith({
uploadFile: {
uploadUrl: 'https://s3.example/presigned',
uploadHeaders,
fileId: 'file-3',
key: 'ghi789',
deduplicated: false,
expiresAt: '2030-01-01T00:00:00Z',
previousVersionId: null,
},
});

await client.upload({ filesTable: 'files' }, { file: fakeFile() });

expect(puts).toEqual([
{ url: 'https://s3.example/presigned', contentType: 'text/plain', size: 11, headers: uploadHeaders },
]);
});

it('refuses to upload without a transport adapter', async () => {
const execute: GraphQLExecutor = async () =>
metaResult() as unknown as Record<string, unknown>;
Expand Down
3 changes: 2 additions & 1 deletion graphql/query/src/storage/document.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import type { StorageSurface } from './types';

const UPLOAD_PAYLOAD_FIELDS = [
'uploadUrl',
'uploadHeaders',
'fileId',
'key',
'deduplicated',
Expand All @@ -24,7 +25,7 @@ const UPLOAD_PAYLOAD_FIELDS = [
*
* mutation UploadAppFileMutation($input: UploadAppFileInput!) {
* uploadAppFile(input: $input) {
* uploadUrl fileId key deduplicated expiresAt previousVersionId
* uploadUrl uploadHeaders fileId key deduplicated expiresAt previousVersionId
* }
* }
*/
Expand Down
5 changes: 4 additions & 1 deletion graphql/query/src/storage/transport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,17 @@ export interface StorageTransport {
/**
* PUT the bytes to a presigned URL. `contentHash` is the hex SHA-256 the
* upload was requested with; the URL is signed over it as
* `x-amz-checksum-sha256`, so the transport must send it.
* `x-amz-checksum-sha256`, so the transport must send it. `headers` are the
* payload's `uploadHeaders` — the exact headers the URL needs (checksum and
* `x-amz-meta-upload-id` included); a transport sends them as-is when given.
*/
putObject(
url: string,
body: ArrayBuffer,
contentType: string,
signal?: AbortSignal,
contentHash?: string,
headers?: Record<string, string>,
): Promise<void>;
}

Expand Down
11 changes: 9 additions & 2 deletions graphql/query/src/storage/upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ export async function uploadToSurface(
}

const payload = data?.[surface.upload.mutation] as
| (StorageUploadResult & { uploadUrl: string | null })
| (StorageUploadResult & { uploadUrl: string | null; uploadHeaders?: Record<string, string> | null })
| undefined;
if (!payload) {
throw new StorageError(
Expand All @@ -114,7 +114,14 @@ export async function uploadToSurface(
}

checkAborted(signal);
await transport.putObject(payload.uploadUrl, await file.arrayBuffer(), contentType, signal, contentHash);
await transport.putObject(
payload.uploadUrl,
await file.arrayBuffer(),
contentType,
signal,
contentHash,
payload.uploadHeaders ?? undefined,
);

return toResult(payload);
}
Expand Down
Loading
Loading