Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,11 @@

import { S3Client } from '@aws-sdk/client-s3';
import { createS3Bucket } from '@constructive-io/s3-utils';
import { createHash } from 'crypto';

import {
CHECKSUM_SHA256_HEADER,
checksumSha256Base64,
deleteS3Object,
generatePresignedGetUrl,
generatePresignedPutUrl,
Expand Down Expand Up @@ -71,17 +74,23 @@ async function uploadToPresignedUrl(
url: string,
body: string,
contentType: string,
extraHeaders: Record<string, string> = {},
): Promise<Response> {
return fetch(url, {
method: 'PUT',
headers: {
'Content-Type': contentType,
'Content-Length': String(Buffer.byteLength(body)),
...extraHeaders,
},
body,
});
}

function sha256Hex(content: string): string {
return createHash('sha256').update(content).digest('hex');
}

/**
* Download content from a presigned GET URL using native fetch.
*/
Expand Down Expand Up @@ -145,6 +154,68 @@ describe('s3-signer integration (RustFS)', () => {
});
});

describe('checksum-bound presigned PUT (content_hash binds the bytes)', () => {
const contentType = 'text/plain';
const real = 'the bytes this hash names';
const forged = 'FORGED bytes, same length'; // same Content-Length as `real`
const hash = sha256Hex(real);

beforeAll(() => {
expect(Buffer.byteLength(forged)).toBe(Buffer.byteLength(real));
});

it('signs x-amz-checksum-sha256 as a header, not a query parameter', async () => {
const url = await generatePresignedPutUrl(s3Config, 'checksum-shape.txt', contentType, Buffer.byteLength(real), 900, hash);
const params = new URL(url).searchParams;
expect(params.get('X-Amz-SignedHeaders')).toContain(CHECKSUM_SHA256_HEADER);
expect(params.has(CHECKSUM_SHA256_HEADER)).toBe(false);
});

it('accepts the bytes that hash to the content hash', async () => {
const key = 'checksum-ok.txt';
const url = await generatePresignedPutUrl(s3Config, key, contentType, Buffer.byteLength(real), 900, hash);
const res = await uploadToPresignedUrl(url, real, contentType, {
[CHECKSUM_SHA256_HEADER]: checksumSha256Base64(hash),
});
expect(res.status).toBe(200);

const head = await headObject(s3Config, key);
expect(head?.checksumSha256).toBe(checksumSha256Base64(hash));
});

it('rejects same-length bytes that do not hash to it (BadDigest)', async () => {
const key = 'checksum-forged.txt';
const url = await generatePresignedPutUrl(s3Config, key, contentType, Buffer.byteLength(real), 900, hash);
const res = await uploadToPresignedUrl(url, forged, contentType, {
[CHECKSUM_SHA256_HEADER]: checksumSha256Base64(hash),
});
expect(res.status).toBe(400);
expect(await res.text()).toContain('BadDigest');
expect(await headObject(s3Config, key)).toBeNull();
});

it('rejects a PUT that omits the checksum header (signature covers it)', async () => {
const key = 'checksum-missing.txt';
const url = await generatePresignedPutUrl(s3Config, key, contentType, Buffer.byteLength(real), 900, hash);
const res = await uploadToPresignedUrl(url, forged, contentType);
expect(res.status).toBe(403);
expect(await headObject(s3Config, key)).toBeNull();
});

it('cannot replace a confirmed object through the same URL with different bytes', async () => {
const key = 'checksum-replay.txt';
const url = await generatePresignedPutUrl(s3Config, key, contentType, Buffer.byteLength(real), 900, hash);
const headers = { [CHECKSUM_SHA256_HEADER]: checksumSha256Base64(hash) };
expect((await uploadToPresignedUrl(url, real, contentType, headers)).status).toBe(200);

expect((await uploadToPresignedUrl(url, forged, contentType, headers)).status).toBe(400);
expect((await uploadToPresignedUrl(url, forged, contentType)).status).toBe(403);

const getUrl = await generatePresignedGetUrl(s3Config, key);
expect((await downloadFromPresignedUrl(getUrl)).body).toBe(real);
});
});

describe('headObject', () => {
const HEAD_KEY = 'test-head-object.json';
const HEAD_CONTENT = JSON.stringify({ test: true });
Expand Down
10 changes: 9 additions & 1 deletion graphile/graphile-presigned-url-plugin/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,15 @@ export {
export { createPresignedUrlPlugin,PresignedUrlPlugin } from './plugin';
export { PresignedUrlPreset } from './preset';
export { describeS3Failure, s3FailureError } from './s3-failure';
export { copyS3Object, deleteS3Object, generatePresignedGetUrl, generatePresignedPutUrl, headObject } from './s3-signer';
export {
CHECKSUM_SHA256_HEADER,
checksumSha256Base64,
copyS3Object,
deleteS3Object,
generatePresignedGetUrl,
generatePresignedPutUrl,
headObject,
} from './s3-signer';
export { clearBucketCache, clearStorageModuleCache, getBucketConfig, loadAllStorageModules,resolveStorageConfigFromCodec, resolveStorageModuleByFileId } from './storage-module-cache';
export type {
BucketConfig,
Expand Down
5 changes: 4 additions & 1 deletion graphile/graphile-presigned-url-plugin/src/plugin.ts
Original file line number Diff line number Diff line change
Expand Up @@ -829,13 +829,16 @@ export async function processSingleFile(
path: derivedPath,
});

// Generate presigned PUT URL
// Generate presigned PUT URL, signed over the client's SHA-256 so the store
// only accepts bytes that hash to the content hash the row (and, for
// content-addressed buckets, the key and any later dedup hit) is named by.
const uploadUrl = await generatePresignedPutUrl(
s3ForDb,
s3Key,
contentType,
size,
storageConfig.uploadUrlExpirySeconds,
contentHash,
);

const expiresAt = new Date(Date.now() + storageConfig.uploadUrlExpirySeconds * 1000).toISOString();
Expand Down
41 changes: 38 additions & 3 deletions graphile/graphile-presigned-url-plugin/src/s3-signer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,39 @@ import type { S3Config } from './types';

const log = new Logger('graphile-presigned-url:s3');

/** The header a checksum-bound presigned PUT must carry (RFC 4648 base64 of the raw digest). */
export const CHECKSUM_SHA256_HEADER = 'x-amz-checksum-sha256';

/**
* Convert a hex-encoded SHA-256 (the `content_hash` clients send) to the
* base64-of-raw-digest form S3 expects in `x-amz-checksum-sha256`.
*/
export function checksumSha256Base64(contentHashHex: string): string {
if (!/^[0-9a-f]{64}$/i.test(contentHashHex)) {
throw new Error('INVALID_CONTENT_HASH_FORMAT: must be a 64-char hex SHA-256');
}
return Buffer.from(contentHashHex, 'hex').toString('base64');
}

/**
* Generate a presigned PUT URL for uploading a file to S3.
*
* The presigned URL is locked to the specific key, content-type, and
* content-length via the signature. The client MUST use these exact values
* when performing the PUT request.
*
* When `contentHash` is given, the SHA-256 is signed into the request as the
* `x-amz-checksum-sha256` header (kept out of the query string, where stores
* ignore it). The store then rejects any body that does not hash to it
* (`BadDigest`), and a PUT without the header fails the signature. This is
* what makes a content-addressed key actually hold the content it names.
*
* @param s3Config - S3 client and bucket configuration
* @param key - S3 object key (content hash or UUID)
* @param contentType - MIME type (locked into the signature)
* @param contentLength - File size in bytes (locked into the signature)
* @param expiresIn - URL expiry in seconds (default: 900 = 15 minutes)
* @param contentHash - Hex SHA-256 the uploaded bytes must hash to
* @returns Presigned PUT URL
*/
export async function generatePresignedPutUrl(
Expand All @@ -33,25 +54,32 @@ export async function generatePresignedPutUrl(
contentType: string,
contentLength: number,
expiresIn: number = 900,
contentHash?: string,
): Promise<string> {
const command = new PutObjectCommand({
Bucket: s3Config.bucket,
Key: key,
ContentType: contentType,
ContentLength: contentLength,
...(contentHash ? { ChecksumSHA256: checksumSha256Base64(contentHash) } : {}),
});

let url: string;
try {
url = await getSignedUrl(s3Config.client as any, command, { expiresIn });
url = await getSignedUrl(s3Config.client as any, command, {
expiresIn,
...(contentHash ? { unhoistableHeaders: new Set([CHECKSUM_SHA256_HEADER]) } : {}),
});
} catch (err) {
throw s3FailureError(
'PRESIGN_PUT_FAILED',
{ endpoint: s3Config.endpoint, bucket: s3Config.bucket, key, contentType },
err,
);
}
log.debug(`Generated presigned PUT URL for key=${key}, contentType=${contentType}, expires=${expiresIn}s`);
log.debug(
`Generated presigned PUT URL for key=${key}, contentType=${contentType}, checksum=${contentHash ? 'sha256' : 'none'}, expires=${expiresIn}s`,
);
return url;
}

Expand Down Expand Up @@ -148,6 +176,11 @@ export async function copyS3Object(
/**
* Check if an object exists in S3 and optionally verify its content-type.
*
* Asks the store for the object's checksum (`ChecksumMode: ENABLED`); it is
* returned as `checksumSha256` (base64 of the raw digest) when the object was
* written through a checksum-bound PUT, so callers can verify the bytes
* behind a key still hash to the content hash it names.
*
* @param s3Config - S3 client and bucket configuration
* @param key - S3 object key
* @param expectedContentType - Expected MIME type (optional)
Expand All @@ -157,12 +190,13 @@ export async function headObject(
s3Config: S3Config,
key: string,
expectedContentType?: string,
): Promise<{ contentType?: string; contentLength?: number } | null> {
): Promise<{ contentType?: string; contentLength?: number; checksumSha256?: string } | null> {
try {
const response = await s3Config.client.send(
new HeadObjectCommand({
Bucket: s3Config.bucket,
Key: key,
ChecksumMode: 'ENABLED',
}),
);

Expand All @@ -175,6 +209,7 @@ export async function headObject(
return {
contentType: response.ContentType,
contentLength: response.ContentLength,
...(response.ChecksumSHA256 ? { checksumSha256: response.ChecksumSHA256 } : {}),
};
} catch (e: any) {
if (e.name === 'NotFound' || e.$metadata?.httpStatusCode === 404) {
Expand Down
7 changes: 6 additions & 1 deletion graphql/query/src/storage/transport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,17 @@ export interface StorageFile {
export interface StorageTransport {
/** SHA-256 the file contents, hex-encoded */
hashFile(file: StorageFile): Promise<string>;
/** PUT the bytes to a presigned URL */
/**
* PUT the bytes to a presigned URL. `contentHash` is the hex SHA-256 the
* upload was requested with; the URL is signed over it as
* `x-amz-checksum-sha256`, so the transport must send it.
*/
putObject(
url: string,
body: ArrayBuffer,
contentType: string,
signal?: AbortSignal,
contentHash?: string,
): Promise<void>;
Comment on lines 31 to 37

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 bug · medium

Optional contentHash lets custom transports break uploads

StorageTransport.putObject accepts contentHash as an optional trailing parameter (graphql/query/src/storage/transport.ts:31), but the server now makes contentHash a required upload input and signs x-amz-checksum-sha256 into every presigned PUT as an unhoistable header (graphile/graphile-presigned-url-plugin/src/s3-signer.ts:71). A pre-existing custom transport that ignores the fifth argument still type-checks but silently sends no header, so its PUTs fail S3 signature validation with an opaque 403. The same optional signature appears in packages/site-deploy/src/types.ts at line 102.

📋 Prompt for AI Agents

In graphql/query/src/storage/transport.ts lines 31-37, change putObject's trailing contentHash?: string to a required contentHash: string, and update the call site in graphql/query/src/storage/upload.ts:117, the default transport in packages/upload-client/src/storage-transport.ts, the PutObject type in packages/site-deploy/src/types.ts:102, and the test fakes in graphql/query/tests/storage.node.test.ts:119. The server signs x-amz-checksum-sha256 into every presigned PUT URL, so a transport that silently drops the hash produces opaque 403 signature errors; requiring the parameter forces third-party implementations to handle it at compile time.

}

Expand Down
2 changes: 1 addition & 1 deletion graphql/query/src/storage/upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ export async function uploadToSurface(
}

checkAborted(signal);
await transport.putObject(payload.uploadUrl, await file.arrayBuffer(), contentType, signal);
await transport.putObject(payload.uploadUrl, await file.arrayBuffer(), contentType, signal, contentHash);

return toResult(payload);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ describe('database-scope upload surface', () => {
});

it('accepts the PUT, into a physical bucket recorded on the tenant row', async () => {
const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType);
const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { contentHash });
expect(putRes.ok).toBe(true);

const stored = await pg.query(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ describe('image / upload / attachment columns', () => {
});
expect(uploaded.body.errors).toBeUndefined();
const { fileId, key, uploadUrl } = uploaded.body.data.uploadAppFile;
expect((await putToPresignedUrl(uploadUrl, content, 'text/plain')).ok).toBe(true);
expect((await putToPresignedUrl(uploadUrl, content, 'text/plain', { contentHash })).ok).toBe(true);
// The confirm worker is not part of this fixture; it is what moves a row
// from `requested` to `uploaded` once it has inspected the object.
await pg.query(`UPDATE "${storageSchema}".app_files SET status = 'uploaded' WHERE id = $1`, [fileId]);
Expand Down
8 changes: 4 additions & 4 deletions graphql/server-test/__tests__/upload.integration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -425,7 +425,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => {
});

it('should accept a PUT to the presigned URL', async () => {
const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType);
const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { contentHash });
expect(putRes.ok).toBe(true);
await confirmUploaded(contentHash);
});
Expand Down Expand Up @@ -468,7 +468,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => {
});

it('should accept a PUT to the presigned URL', async () => {
const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType);
const putRes = await putToPresignedUrl(uploadUrl, fileContent, contentType, { contentHash });
expect(putRes.ok).toBe(true);
});
});
Expand Down Expand Up @@ -524,7 +524,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => {
expect(retry.uploadUrl).toBeTruthy();
expect(retry.fileId).not.toBe(first.fileId);

const putRes = await putToPresignedUrl(retry.uploadUrl, fileContent, 'text/plain');
const putRes = await putToPresignedUrl(retry.uploadUrl, fileContent, 'text/plain', { contentHash });
expect(putRes.ok).toBe(true);
await confirmUploaded(contentHash);

Expand Down Expand Up @@ -869,7 +869,7 @@ describe('Integration tests (uploads, tenant isolation, RLS)', () => {
expect(payload.fileId).toBeTruthy();
expect(payload.uploadUrl).toBeTruthy();

const putRes = await putToPresignedUrl(payload.uploadUrl, fileContent, 'text/plain');
const putRes = await putToPresignedUrl(payload.uploadUrl, fileContent, 'text/plain', { contentHash });
expect(putRes.ok).toBe(true);
});

Expand Down
2 changes: 2 additions & 0 deletions packages/bucket-provisioner/__tests__/cors.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ describe('buildUploadCorsRules', () => {
expect(rule.allowedHeaders).toContain('Content-Type');
expect(rule.allowedHeaders).toContain('Content-Length');
expect(rule.allowedHeaders).toContain('Authorization');
expect(rule.allowedHeaders).toContain('x-amz-checksum-sha256');
});

it('exposes ETag and Content-Length', () => {
Expand Down Expand Up @@ -74,6 +75,7 @@ describe('buildPrivateCorsRules', () => {
expect(rule.allowedHeaders).toContain('Content-Type');
expect(rule.allowedHeaders).toContain('Content-Length');
expect(rule.allowedHeaders).toContain('Authorization');
expect(rule.allowedHeaders).toContain('x-amz-checksum-sha256');
});

it('uses default maxAgeSeconds of 3600', () => {
Expand Down
2 changes: 2 additions & 0 deletions packages/bucket-provisioner/src/cors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ export function buildUploadCorsRules(
'Content-Length',
'Content-MD5',
'x-amz-content-sha256',
'x-amz-checksum-sha256',
'x-amz-date',
'x-amz-security-token',
'Authorization',
Expand Down Expand Up @@ -81,6 +82,7 @@ export function buildPrivateCorsRules(
'Content-Length',
'Content-MD5',
'x-amz-content-sha256',
'x-amz-checksum-sha256',
'x-amz-date',
'x-amz-security-token',
'Authorization',
Expand Down
6 changes: 3 additions & 3 deletions packages/site-deploy/src/cas-upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ export async function uploadMissing(
if (index >= diff.missing.length) return;
const { file, uploadUrl } = diff.missing[index];
await withRetries(
() => put(uploadUrl, file.bytes, file.contentType, options.signal),
() => put(uploadUrl, file.bytes, file.contentType, options.signal, file.hash),
options.retries,
file.path,
options.signal,
Expand All @@ -163,11 +163,11 @@ export async function uploadMissing(
};
}

const defaultPut: PutObject = async (url, body, contentType, signal) => {
const defaultPut: PutObject = async (url, body, contentType, signal, contentHash) => {
// A fresh copy: putToPresignedUrl takes a BodyInit, and a Uint8Array view
// over a pooled Buffer would send the whole pool.
const buffer = body.slice().buffer as ArrayBuffer;
await putToPresignedUrl(url, buffer, contentType, signal);
await putToPresignedUrl(url, buffer, contentType, { signal, contentHash });
};

async function withRetries(
Expand Down
2 changes: 2 additions & 0 deletions packages/site-deploy/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,8 @@ export type PutObject = (
body: Uint8Array,
contentType: string,
signal?: AbortSignal,
/** Hex SHA-256 of `body`; the presigned URL is signed over it as `x-amz-checksum-sha256`. */
contentHash?: string,
) => Promise<void>;

export interface DeploySiteOptions {
Expand Down
Loading
Loading