diff --git a/README.md b/README.md index a53ea10..207f9dd 100644 --- a/README.md +++ b/README.md @@ -339,6 +339,35 @@ const { stream, status, headers } = await proxy.fetch({ return new Response(Readable.toWeb(stream) as ReadableStream, { status, headers }); ``` +#### Custom URL → key mapping (`fetchWebKey`) + +When the S3 key doesn't come straight from the request URL — a lookup table, a +database id, a key you've already resolved — use +**`proxy.fetchWebKey(key, request)`**. The caller names the key; the `Request` +still supplies the method and `Range` header, and you get the same `Response` +with the same HTTP translation as `fetchWeb`: + +```typescript +// The key comes from your own mapping, not the URL path. +app.get('/download/:id', async (c) => { + const key = await resolveS3Key(c.req.param('id')); // → the exact S3 object key + return proxy.fetchWebKey(key, c.req.raw); +}); +``` + +Because you pass the key directly it is used **verbatim** — s3proxy never +re-parses it out of a URL. That sidesteps the subtle bug where a key containing +`%`, `+`, spaces, or other reserved characters serves the wrong object (or 404s) +because a URL-encode step and s3proxy's decode step disagree. (If you *do* +derive the key from the URL path, you still own decoding it correctly — note +that some frameworks pre-decode their path accessor, e.g. Hono's `c.req.path`; +read the raw URL when the exact bytes matter. For the identity case, just use +`fetchWeb`.) + +> If you don't need an HTTP `Response` at all (backend processing, ETL, an API +> shaping its own headers), use the AWS SDK directly — s3proxy adds nothing +> there. Its job is serving S3 objects over HTTP, correctly. + > **Performance note.** On Node, Hono goes through `@hono/node-server` > which converts `IncomingMessage` and `Web Request/Response` on every > request, two extra layers compared with Express's direct @@ -507,6 +536,15 @@ so your framework's error handler owns the error-body format. HEAD responses have a `null` body. Uses the runtime's global `Request` / `Response`; no extra dependency. +##### `await proxy.fetchWebKey(key: string, request: Request): Promise` +Key-addressed sibling of `fetchWeb`: the caller supplies the S3 `key` and the +`Request` contributes only the method and `Range` header — the URL is not +consulted for key selection. Same HTTP translation and typed-error behavior as +`fetchWeb`. Use it when the key doesn't come from the URL path (a lookup, a +prefix, a mount point); the `key` is used verbatim, avoiding the URL +encode/decode round-trip that corrupts keys containing `%`, `+`, spaces, or +other reserved characters. + ##### `await proxy.healthCheck(): Promise` Verify bucket connectivity. Resolves on success, throws a typed `S3ProxyError` on failure. Use it from a `/health` or `/ready` handler diff --git a/src/index.ts b/src/index.ts index ff8f805..48ec0ab 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,7 @@ import { EventEmitter } from 'node:events'; import { pipeline, Readable } from 'node:stream'; import { GetObjectCommand, HeadBucketCommand, HeadObjectCommand } from '@aws-sdk/client-s3'; -import { S3Forbidden, S3NotFound, S3ProxyError } from './errors.js'; +import { InvalidRequest, S3Forbidden, S3NotFound, S3ProxyError } from './errors.js'; import { mapHeaderToParam, parseRequest } from './request-parser.js'; import { S3Gateway } from './s3-gateway.js'; import type { @@ -102,15 +102,56 @@ export class S3Proxy extends EventEmitter { * `Response` yourself. */ public async fetchWeb(request: Request): Promise { - const { stream, status, headers } = await this.fetch({ + // URL picks the key: derive it from the request path (percent-decoded by + // parseRequest), then hand off to the key-addressed seam. Key derivation + // reads only the path — method, headers, and the query string play no role + // (`url` is required by the HttpRequest type but unused when `path` is set). + const key = parseRequest({ url: request.url, - // Pass the parsed pathname as `path` so parseRequest uses it directly - // rather than re-parsing the URL. The S3 key derives from the path - // alone, so the query string is irrelevant here. path: new URL(request.url).pathname, + headers: {}, + }).key; + return this.fetchWebKey(key, request); + } + + /** + * Adapt a WHATWG `Request` to the internal `HttpRequest` shape. Shared by the + * two web entry points so the method/header lift lives in one place. + */ + private webToHttpRequest(request: Request): HttpRequest { + return { + url: request.url, method: request.method, headers: Object.fromEntries(request.headers), - }); + }; + } + + /** + * Key-addressed sibling of `fetchWeb`: the *caller* names the S3 key and the + * `Request` contributes only the method (GET/HEAD) and the `Range` header — + * the request URL plays no role in key selection. Returns the same + * Web `Response` with the same HTTP <-> S3 translation as `fetchWeb` + * (`Range` -> 206, missing/denied -> typed 404/403, header mapping, HEAD + * semantics, Node -> Web stream), and likewise throws the typed + * `S3ProxyError` on classified failures rather than rendering them. + * + * Use this when your URL -> key mapping is not identity (e.g. a bucket + * mounted under `/files/*`). It avoids round-tripping the key through URL + * encode/decode, which otherwise forces your encoder and s3proxy's decoder to + * agree exactly for keys containing `%`, `+`, spaces, or other reserved + * characters — or the wrong object is served / a valid key 404s. The key is + * used verbatim as the S3 object key. + */ + public async fetchWebKey(key: string, request: Request): Promise { + // The key is used verbatim (never decoded); still reject the null byte S3 + // rejects and that every other entry point guards via parseRequest, so the + // typed-error contract (InvalidRequest, not a generic 500) holds here too. + if (key.includes('\0')) { + throw new InvalidRequest('null byte in key'); + } + // fetchKey consumes only the method and Range header; the `url` carried by + // webToHttpRequest is unused for a key-addressed call. + const { stream, status, headers } = await this.fetchKey(key, this.webToHttpRequest(request)); // No body for HEAD; otherwise adapt the Node stream to a Web stream. // Backpressure propagates through the wrapper, so nothing buffers. const body = request.method === 'HEAD' ? null : (Readable.toWeb(stream) as ReadableStream); diff --git a/test/fetch-web.test.ts b/test/fetch-web.test.ts index e99232a..2335af8 100644 --- a/test/fetch-web.test.ts +++ b/test/fetch-web.test.ts @@ -1,24 +1,25 @@ import { GetObjectCommand, S3ServiceException } from '@aws-sdk/client-s3'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { S3Forbidden, S3NotFound, S3Proxy } from '../src/index.js'; +import { InvalidRequest, S3Forbidden, S3NotFound, S3Proxy } from '../src/index.js'; import { s3Mock, setupS3Mocks, teardownS3Mocks } from './helpers/aws-mock.js'; import { catchError } from './helpers/http-mocks.js'; const url = (path: string) => `http://localhost${path}`; -describe('proxy.fetchWeb', () => { - let proxy: S3Proxy; +// Shared setup for both suites below: a fresh mocked proxy per test. +let proxy: S3Proxy; - beforeEach(async () => { - setupS3Mocks(); - proxy = new S3Proxy({ bucket: '.test-bucket' }); - await proxy.init(); - }); +beforeEach(async () => { + setupS3Mocks(); + proxy = new S3Proxy({ bucket: '.test-bucket' }); + await proxy.init(); +}); - afterEach(() => { - teardownS3Mocks(); - }); +afterEach(() => { + teardownS3Mocks(); +}); +describe('proxy.fetchWeb', () => { it('returns a 200 Web Response with headers and a streamed body for GET', async () => { const res = await proxy.fetchWeb(new Request(url('/index.html'))); expect(res).toBeInstanceOf(Response); @@ -69,3 +70,67 @@ describe('proxy.fetchWeb', () => { expect(err).toBeInstanceOf(S3Forbidden); }); }); + +describe('proxy.fetchWebKey', () => { + it('serves the caller-supplied key, ignoring the request URL', async () => { + // URL path is "/" but the key argument selects index.html. + const res = await proxy.fetchWebKey('index.html', new Request(url('/'))); + expect(res).toBeInstanceOf(Response); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('text/html'); + expect((await res.text()).length).toBe(338); + }); + + it('is behaviorally identical to fetchWeb when the key matches the path', async () => { + const viaKey = await proxy.fetchWebKey('index.html', new Request(url('/ignored'))); + const viaUrl = await proxy.fetchWeb(new Request(url('/index.html'))); + expect(viaKey.status).toBe(viaUrl.status); + expect(viaKey.headers.get('content-type')).toBe(viaUrl.headers.get('content-type')); + expect(viaKey.headers.get('content-length')).toBe(viaUrl.headers.get('content-length')); + }); + + it('honors the Range header from the request → 206 + content-range', async () => { + const res = await proxy.fetchWebKey( + 'index.html', + new Request(url('/'), { headers: { range: 'bytes=0-100' } }) + ); + expect(res.status).toBe(206); + expect(res.headers.get('content-range')).toBe('bytes 0-100/338'); + expect((await res.text()).length).toBe(101); + }); + + it('returns a null body for HEAD but keeps status and headers', async () => { + const res = await proxy.fetchWebKey('index.html', new Request(url('/'), { method: 'HEAD' })); + expect(res.status).toBe(200); + expect(res.body).toBeNull(); + expect(res.headers.get('content-length')).toBe('338'); + }); + + it('throws the typed S3NotFound for a missing key', async () => { + await expect( + proxy.fetchWebKey('nonexistent-file.txt', new Request(url('/'))) + ).rejects.toBeInstanceOf(S3NotFound); + }); + + // The point of the seam: a key with a literal '%' is served verbatim, where + // routing it through a URL (fetchWeb) fails because the path can't be decoded. + it('serves a key with reserved characters verbatim', async () => { + const res = await proxy.fetchWebKey('reports/q1 50%.txt', new Request(url('/'))); + expect(res.status).toBe(200); + expect(await res.text()).toBe('fifty percent'); + }); + + it('contrast: the same key via fetchWeb (URL-addressed) fails to decode', async () => { + await expect(proxy.fetchWeb(new Request(url('/reports/q1 50%.txt')))).rejects.toBeInstanceOf( + InvalidRequest + ); + }); + + it('rejects a key containing a null byte with the typed InvalidRequest', async () => { + // Matches the guard parseRequest applies on the URL-addressed entry points, + // so the typed-error contract is consistent across the API. + await expect( + proxy.fetchWebKey('reports/bad\0key.txt', new Request(url('/'))) + ).rejects.toBeInstanceOf(InvalidRequest); + }); +}); diff --git a/test/helpers/aws-mock.ts b/test/helpers/aws-mock.ts index 80ff1e0..512aa1e 100644 --- a/test/helpers/aws-mock.ts +++ b/test/helpers/aws-mock.ts @@ -217,6 +217,16 @@ The public repo is here. }, }); + // Mock an object whose key contains a literal '%' — a key the URL-addressed + // path can't carry safely (decodeURIComponent on the path throws), but the + // key-addressed seam (fetchWebKey) serves verbatim. + s3Mock.on(GetObjectCommand, { Bucket: '.test-bucket', Key: 'reports/q1 50%.txt' }).resolves({ + Body: mockBody('fifty percent'), + ContentLength: 13, + ContentType: 'text/plain', + $metadata: { httpStatusCode: 200, requestId: 'mock-request-id' }, + }); + // Mock range requests for large.bin (for MockExpress test) s3Mock .on(GetObjectCommand, {