From 24d837546fe6cc387cd9e75d1c0db893c1268002 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 1 Nov 2026 11:14:59 -0800 Subject: [PATCH 1/2] fix(public-shares): stream shared files with byte ranges instead of buffering them under 100 MB --- .../api/files/public/[token]/content/route.ts | 11 +- apps/sim/hooks/use-file-content-source.tsx | 17 +- .../content-sharing.integration.ts | 163 ++++++++++++++---- .../sim/lib/public-shares/application/read.ts | 149 +++++++++++++--- 4 files changed, 275 insertions(+), 65 deletions(-) diff --git a/apps/sim/app/api/files/public/[token]/content/route.ts b/apps/sim/app/api/files/public/[token]/content/route.ts index b85fe6df606..e665b3a474c 100644 --- a/apps/sim/app/api/files/public/[token]/content/route.ts +++ b/apps/sim/app/api/files/public/[token]/content/route.ts @@ -17,7 +17,10 @@ import { } from '@/lib/public-shares/application' import { enforcePublicFileRateLimit } from '@/lib/public-shares/rate-limit' import { FILE_CACHE_CONTROL } from '@/lib/uploads/server/delivery' -import { createConditionalFileResponse } from '@/app/api/files/utils' +import { + createConditionalFileResponse, + createStoredFileStreamResponse, +} from '@/app/api/files/utils' export const dynamic = 'force-dynamic' const logger = createLogger('PublicFileContentAPI') @@ -49,6 +52,12 @@ export const GET = withRouteHandler( preview: parsed.data.query.preview === '1', request, }) + if (result.kind === 'stream') { + return await createStoredFileStreamResponse( + { ...result, cacheControl: FILE_CACHE_CONTROL.revalidate }, + request.headers + ) + } return createConditionalFileResponse( { buffer: result.buffer, diff --git a/apps/sim/hooks/use-file-content-source.tsx b/apps/sim/hooks/use-file-content-source.tsx index 9dd838cc9b6..803e54dcee3 100644 --- a/apps/sim/hooks/use-file-content-source.tsx +++ b/apps/sim/hooks/use-file-content-source.tsx @@ -146,12 +146,17 @@ export function createPublicFileContentSource( contentUrl: string, owner?: EditableFileOwner ): FileContentSource { - return inlineImageSource( - (_key, opts) => - opts?.preview ? `${contentUrl}${contentUrl.includes('?') ? '&' : '?'}preview=1` : contentUrl, - `/api/files/public/${token}/inline`, - owner - ) + return { + ...inlineImageSource( + (_key, opts) => + opts?.preview + ? `${contentUrl}${contentUrl.includes('?') ? '&' : '?'}preview=1` + : contentUrl, + `/api/files/public/${token}/inline`, + owner + ), + servesByteRanges: true, + } } /** diff --git a/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts b/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts index 4dc2fbaf745..06ba429339c 100644 --- a/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts +++ b/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts @@ -1,4 +1,4 @@ -import { mkdtempSync } from 'node:fs' +import { existsSync, mkdtempSync } from 'node:fs' import { mkdir, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' @@ -39,8 +39,11 @@ import { createProjectFile, updateProjectFileContent, } from '@/lib/projects/files/application/content' +import type { readPublicFileShareContent } from '@/lib/public-shares/application' import * as shareManager from '@/lib/public-shares/manager' import * as storage from '@/lib/uploads/core/storage-service' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { sanitizeFileKey } from '@/lib/uploads/utils/file' vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) @@ -129,6 +132,16 @@ async function fixture() { return { ...ids, principal: createSessionPrincipal({ userId: editorId }) } } +type PublicShareContent = Awaited> + +/** The bytes a public read delivers, whether streamed from storage or built whole. */ +async function sharedBytes(read: PublicShareContent | Promise) { + const result = await read + return result.kind === 'stream' + ? Buffer.from(await new Response(await result.open(undefined)).arrayBuffer()) + : result.buffer +} + function createInput(projectId: string, content = 'Project architecture') { return { projectId, @@ -410,9 +423,9 @@ describe('Public file shares against PostgreSQL and private storage', () => { credential: { method: 'POST', password: 'correct-password' }, }) if (!accepted.authorized) throw new Error('Correct share password rejected') - expect((await readPublicFileShareContent({ grant: accepted.grant })).buffer.toString()).toBe( - 'Shared architecture' - ) + expect( + (await sharedBytes(readPublicFileShareContent({ grant: accepted.grant }))).toString() + ).toBe('Shared architecture') const [passwordShare] = await db .select() .from(publicShare) @@ -653,6 +666,74 @@ describe('Public file shares against PostgreSQL and private storage', () => { } ) + check( + 'public share streams stored bytes past the buffered ceiling and refuses once the share is disabled', + async () => { + const { updateProjectFileShare } = await import('@/lib/projects/files/application/shares') + const { authorizePublicFileShare, readPublicFileShareContent } = await import( + '@/lib/public-shares/application' + ) + const f = await fixture() + const share = async (name: string, contentType: string) => { + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: { ...createInput(f.projectId, 'placeholder'), name, contentType }, + }) + const { share } = await updateProjectFileShare.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: file.id, isActive: true }, + }) + const access = await authorizePublicFileShare({ + token: share.token, + credential: { method: 'GET' }, + }) + if (!access.authorized) throw new Error(`Public ${name} fixture rejected`) + return { file, grant: access.grant } + } + + const recording = await share('recording.mp4', 'video/mp4') + const size = MAX_BUFFERED_TRANSFER_BYTES + 1024 + const bytes = Buffer.alloc(size) + bytes.write('tail', size - 4) + const [row] = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.id, recording.file.id)) + const objectPath = join(localStorageRoot, sanitizeFileKey(row.key)) + expect(existsSync(objectPath)).toBe(true) + await writeFile(objectPath, bytes) + await db + .update(workspaceFiles) + .set({ sizeBytes: size }) + .where(eq(workspaceFiles.id, recording.file.id)) + + const stored = await readPublicFileShareContent({ grant: recording.grant }) + if (stored.kind !== 'stream') throw new Error('A stored recording was buffered') + expect(stored.size).toBe(size) + const whole = Buffer.from(await new Response(await stored.open(undefined)).arrayBuffer()) + expect(whole.length).toBe(size) + expect(whole.subarray(size - 4).toString()).toBe('tail') + const span = await stored.open({ start: size - 4, end: size - 1 }) + expect(Buffer.from(await new Response(span).arrayBuffer()).toString()).toBe('tail') + + await updateProjectFileShare.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: recording.file.id, isActive: false }, + }) + await expect(stored.open({ start: 0, end: 3 })).rejects.toMatchObject({ code: 'not_found' }) + + const page = await share('notes.html', 'text/html') + expect(await readPublicFileShareContent({ grant: page.grant })).toMatchObject({ + kind: 'buffer', + }) + // A generation source is compiled, never streamed: with no cached artifact it refuses. + const source = await share('report.pptx', 'text/x-pptxgenjs') + await expect(readPublicFileShareContent({ grant: source.grant })).rejects.toMatchObject({ + code: 'conflict', + }) + } + ) + check( 'public share Office artifacts require the current owner-qualified dependency cache without compilation', async () => { @@ -693,7 +774,7 @@ describe('Public file shares against PostgreSQL and private storage', () => { principal: f.principal, input: { projectId: f.projectId, fileId: source.file.id, maxBytes: 1024 }, }) - expect((await readPublicFileShareContent({ grant: access.grant })).buffer).toEqual( + expect(await sharedBytes(readPublicFileShareContent({ grant: access.grant }))).toEqual( rendered.buffer ) await updateProjectFileContent.execute({ @@ -740,37 +821,49 @@ describe('Public file shares against PostgreSQL and private storage', () => { credential: { method: 'POST', password: 'initial-password' }, }) if (!access.authorized) throw new Error('Initial credential rejected') + const applyChange = async () => { + if (change === 'disabled') + await db + .update(publicShare) + .set({ isActive: false }) + .where(eq(publicShare.id, share.id)) + if (change === 'password') + await db + .update(publicShare) + .set({ password: (await encryptSecret('replacement-password')).encrypted }) + .where(eq(publicShare.id, share.id)) + if (change === 'provenance') + await db + .update(workspaceFileSecretProvenance) + .set({ status: 'unknown' }) + .where(eq(workspaceFileSecretProvenance.fileId, file.id)) + if (change === 'archived') + await db + .update(workspaceFiles) + .set({ deletedAt: new Date() }) + .where(eq(workspaceFiles.id, file.id)) + } + // The change lands while the object is being read, buffered or streamed alike. const download = storage.downloadFile - const spy = vi.spyOn(storage, 'downloadFile').mockImplementation(async (options) => { - const result = await download(options) - if (options.key === file.key) { - if (change === 'disabled') - await db - .update(publicShare) - .set({ isActive: false }) - .where(eq(publicShare.id, share.id)) - if (change === 'password') - await db - .update(publicShare) - .set({ password: (await encryptSecret('replacement-password')).encrypted }) - .where(eq(publicShare.id, share.id)) - if (change === 'provenance') - await db - .update(workspaceFileSecretProvenance) - .set({ status: 'unknown' }) - .where(eq(workspaceFileSecretProvenance.fileId, file.id)) - if (change === 'archived') - await db - .update(workspaceFiles) - .set({ deletedAt: new Date() }) - .where(eq(workspaceFiles.id, file.id)) - } - return result - }) - await expect(readPublicFileShareContent({ grant: access.grant })).rejects.toMatchObject({ + const downloadStream = storage.downloadFileStream + const spies = [ + vi.spyOn(storage, 'downloadFile').mockImplementation(async (options) => { + const result = await download(options) + if (options.key === file.key) await applyChange() + return result + }), + vi.spyOn(storage, 'downloadFileStream').mockImplementation(async (options) => { + const result = await downloadStream(options) + if (options.key === file.key) await applyChange() + return result + }), + ] + await expect( + sharedBytes(readPublicFileShareContent({ grant: access.grant })) + ).rejects.toMatchObject({ code: change === 'provenance' ? 'forbidden' : 'not_found', }) - spy.mockRestore() + for (const spy of spies) spy.mockRestore() } } ) @@ -836,7 +929,7 @@ describe('Public file shares against PostgreSQL and private storage', () => { }) if (!legacyAccess.authorized) throw new Error('Existing workspace share rejected') expect( - (await readPublicFileShareContent({ grant: legacyAccess.grant })).buffer.toString() + (await sharedBytes(readPublicFileShareContent({ grant: legacyAccess.grant }))).toString() ).toBe('Workspace link') } ) @@ -892,7 +985,7 @@ describe('Public file shares against PostgreSQL and private storage', () => { if (!access.authorized) throw new Error('Public page rejected') const rendered = await readPublicFileShareContent({ grant: access.grant }) expect(rendered.contentType).toBe('text/html') - expect(rendered.buffer.toString()).toContain( + expect((await sharedBytes(rendered)).toString()).toContain( `data:image/png;base64,${image.toString('base64')}` ) expect( diff --git a/apps/sim/lib/public-shares/application/read.ts b/apps/sim/lib/public-shares/application/read.ts index 9b881e76820..9248e77adac 100644 --- a/apps/sim/lib/public-shares/application/read.ts +++ b/apps/sim/lib/public-shares/application/read.ts @@ -7,6 +7,7 @@ import { OrchestrationError, type OrchestrationRequestContext, } from '@/lib/core/orchestration/types' +import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import type { DbTransaction } from '@/lib/db/types' import { @@ -23,16 +24,21 @@ import { mergeWorkspaceFileSecretProvenance, type WorkspaceFileSecretProvenance, } from '@/lib/uploads/contexts/workspace/file-secret-provenance' -import { downloadFile, headObject } from '@/lib/uploads/core/storage-service' +import { downloadFile, downloadFileStream, headObject } from '@/lib/uploads/core/storage-service' import { collectReferencedFileIds, getDocumentSourceLanguage, getE2BDocFormat, isCompiledDocumentBuffer, } from '@/lib/uploads/documents/compile' +import { storedObjectEtag } from '@/lib/uploads/server/delivery' import { hasEmbeddedFileRef } from '@/lib/uploads/server/embedded-image-refs' import { resolveServableImageBytes } from '@/lib/uploads/server/image-derivative' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { + type ByteRange, + getWorkspaceFileSize, + MAX_BUFFERED_TRANSFER_BYTES, +} from '@/lib/uploads/shared/types' import { resolveEffectiveMimeType } from '@/lib/uploads/utils/file' import { sniffImageContentType } from '@/lib/uploads/utils/validation' import { reportWorkspaceFileDelivery } from '@/lib/workspace-files/application/file-delivery-observer' @@ -44,6 +50,7 @@ import { collectSimPageFileReferences, inlineSimPageImages, } from '@/lib/workspace-files/page-document.server' +import { servesStoredBytes } from '@/lib/workspace-files/stored-bytes' interface PublicReadInput { grant: VerifiedPublicFileShareGrant @@ -127,6 +134,39 @@ async function dependencies( await admitPublicFiles(tx, rows) return rows } +/** Audits one anonymous delivery of a shared file, or of a document input it embedded. */ +function recordPublicShareDownload(input: { + workspaceId: string | null + organizationId: string | null + owner: PublicFileShareSnapshot['owner'] + sharedFile: WorkspaceFileRow + servedFile: WorkspaceFileRow + operation: PublicFileReadOperation + bytes: number + request?: OrchestrationRequestContext +}) { + const { servedFile } = input + recordAudit({ + actorId: null, + workspaceId: input.workspaceId, + action: AuditAction.FILE_DOWNLOADED, + resourceType: AuditResourceType.FILE, + resourceId: servedFile.id, + resourceName: servedFile.originalName, + description: `Public share download of "${servedFile.originalName}"`, + request: input.request, + metadata: { + organizationId: input.organizationId, + operation: input.operation.id, + sharedDocumentId: input.sharedFile.id, + sharedByUserId: input.sharedFile.userId, + access: 'public_share', + anonymous: true, + owner: input.owner, + bytes: input.bytes, + }, + }) +} async function finish( grant: VerifiedPublicFileShareGrant, manifest: PublicManifest, @@ -165,25 +205,15 @@ async function finish( } }) await reportWorkspaceFileDelivery(result.secretProvenance, servedFile.id) - recordAudit({ - actorId: null, + recordPublicShareDownload({ workspaceId: manifest.snapshot.workspaceId, - action: AuditAction.FILE_DOWNLOADED, - resourceType: AuditResourceType.FILE, - resourceId: servedFile.id, - resourceName: servedFile.originalName, - description: `Public share download of "${servedFile.originalName}"`, + organizationId: result.organizationId, + owner: result.owner, + sharedFile: manifest.snapshot.file, + servedFile, + operation, + bytes: buffer.length, request, - metadata: { - organizationId: result.organizationId, - operation: operation.id, - sharedDocumentId: result.file.id, - sharedByUserId: manifest.snapshot.file.userId, - access: 'public_share', - anonymous: true, - owner: result.owner, - bytes: buffer.length, - }, }) return result } @@ -207,8 +237,10 @@ export async function checkPublicFileShareContent( }) const stored = await headObject(initial.file.key, initial.adapter.storageContext) if (!stored) throw new OrchestrationError('not_found', 'File not found') - assertKnownSizeWithinLimit(stored.size, MAX_BUFFERED_TRANSFER_BYTES, 'File') - const format = await getE2BDocFormat(initial.file.originalName) + // Stored bytes stream at any size; only a representation built from them is held whole. + const streamed = servesStoredBytes(initial.file.contentType, initial.file.originalName) + if (!streamed) assertKnownSizeWithinLimit(stored.size, MAX_BUFFERED_TRANSFER_BYTES, 'File') + const format = streamed ? null : await getE2BDocFormat(initial.file.originalName) let inputs: WorkspaceFileRow[] = [] if (format) { const source = await downloadFile({ @@ -264,7 +296,72 @@ export async function checkPublicFileShareContent( } } -/** A public read never executes generated document code; it serves current cached artifacts only. */ +/** + * A shared file served as its stored bytes, read from storage only as far as a response needs: + * the whole object, or one inclusive span. The bytes are never held, so a file of any size is + * served and a byte range is read on its own. + * + * `open` rechecks the grant, the revision, and the file's provenance once storage has opened and + * before the stream is handed out, as a buffered read does after rendering. One view is audited + * once: a download, or the first span of a player's ranged reads. + */ +function streamSharedFile( + grant: VerifiedPublicFileShareGrant, + initial: PublicFileShareSnapshot, + operation: PublicFileReadOperation, + request?: OrchestrationRequestContext +) { + const { file } = initial + const size = getWorkspaceFileSize(file) + return { + kind: 'stream' as const, + filename: file.originalName, + contentType: resolveEffectiveMimeType(undefined, file.originalName), + size, + /** A content write uploads under a fresh key and stamps `contentUpdatedAt`. */ + etag: storedObjectEtag([file.key, size, file.contentUpdatedAt.getTime()]), + async open(range: ByteRange | undefined): Promise> { + const stream = nodeReadableToWebStream( + await downloadFileStream({ key: file.key, context: initial.adapter.storageContext, range }) + ) + try { + const provenance = await withPublicFileShareGrant( + grant, + operation, + async (tx, snapshot) => { + requireRevision(snapshot.file, file) + return admitPublicFiles(tx, [snapshot.file]) + } + ) + await reportWorkspaceFileDelivery(provenance, file.id) + } catch (error) { + await stream.cancel(error).catch(() => undefined) + throw asOrchestrationError(error) ?? error + } + if (!range || range.start === 0) { + recordPublicShareDownload({ + workspaceId: initial.workspaceId, + organizationId: initial.organizationId, + owner: initial.owner, + sharedFile: file, + servedFile: file, + operation, + bytes: range ? range.end - range.start + 1 : size, + request, + }) + } + return stream + }, + } +} + +/** + * Reads a shared file for delivery. A file served as its stored bytes comes back as a stream of any + * size (see {@link servesStoredBytes}); a page or generated document is rendered, and a `preview` + * may be a HEIC transcode sniffed from the bytes, so those are built whole under the buffered + * ceiling. A public read never executes generated document code; it serves current cached + * artifacts only. + */ export async function readPublicFileShareContent({ grant, preview, @@ -276,6 +373,9 @@ export async function readPublicFileShareContent({ await admitPublicFiles(tx, [snapshot.file]) return snapshot }) + if (!preview && servesStoredBytes(initial.file.contentType, initial.file.originalName)) { + return streamSharedFile(grant, initial, operation, request) + } const raw = await downloadFile({ key: initial.file.key, context: initial.adapter.storageContext, @@ -361,7 +461,10 @@ export async function readPublicFileShareContent({ const image = await resolveServableImageBytes(raw, initial.file.key, { owner: initial.owner }) if (image) ({ buffer, contentType } = image) } - return await finish(grant, manifest, operation, buffer, contentType, request) + return { + kind: 'buffer' as const, + ...(await finish(grant, manifest, operation, buffer, contentType, request)), + } } catch (error) { throw asOrchestrationError(error) ?? error } From b4c2de7667cacc900b6818cfe08a1f4d33b1f7b8 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 1 Nov 2026 11:26:18 -0800 Subject: [PATCH 2/2] fix(public-shares): size a streamed share from storage, audit every delivered response, cover both read paths --- .../content-sharing.integration.ts | 140 ++++++++++-------- .../sim/lib/public-shares/application/read.ts | 52 +++---- .../scripts/test-project-file-public-e2e.ts | 17 ++- 3 files changed, 116 insertions(+), 93 deletions(-) diff --git a/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts b/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts index 06ba429339c..841573b0f67 100644 --- a/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts +++ b/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts @@ -800,71 +800,83 @@ describe('Public file shares against PostgreSQL and private storage', () => { '@/lib/public-shares/application' ) const { encryptSecret } = await import('@/lib/core/security/encryption') - for (const change of ['disabled', 'password', 'archived', 'provenance'] as const) { - const f = await fixture() - const { file } = await createProjectFile.execute({ - principal: f.principal, - input: createInput(f.projectId), - }) - const { share } = await updateProjectFileShare.execute({ - principal: f.principal, - input: { - projectId: f.projectId, - fileId: file.id, - isActive: true, - authType: 'password', - password: 'initial-password', - }, - }) - const access = await authorizePublicFileShare({ - token: share.token, - credential: { method: 'POST', password: 'initial-password' }, - }) - if (!access.authorized) throw new Error('Initial credential rejected') - const applyChange = async () => { - if (change === 'disabled') - await db - .update(publicShare) - .set({ isActive: false }) - .where(eq(publicShare.id, share.id)) - if (change === 'password') - await db - .update(publicShare) - .set({ password: (await encryptSecret('replacement-password')).encrypted }) - .where(eq(publicShare.id, share.id)) - if (change === 'provenance') - await db - .update(workspaceFileSecretProvenance) - .set({ status: 'unknown' }) - .where(eq(workspaceFileSecretProvenance.fileId, file.id)) - if (change === 'archived') - await db - .update(workspaceFiles) - .set({ deletedAt: new Date() }) - .where(eq(workspaceFiles.id, file.id)) - } - // The change lands while the object is being read, buffered or streamed alike. - const download = storage.downloadFile - const downloadStream = storage.downloadFileStream - const spies = [ - vi.spyOn(storage, 'downloadFile').mockImplementation(async (options) => { - const result = await download(options) - if (options.key === file.key) await applyChange() - return result - }), - vi.spyOn(storage, 'downloadFileStream').mockImplementation(async (options) => { - const result = await downloadStream(options) - if (options.key === file.key) await applyChange() - return result - }), - ] - await expect( - sharedBytes(readPublicFileShareContent({ grant: access.grant })) - ).rejects.toMatchObject({ - code: change === 'provenance' ? 'forbidden' : 'not_found', - }) - for (const spy of spies) spy.mockRestore() + // A Markdown file streams from storage; a page is rendered whole. Both reads must refuse. + const shared = { + streamed: (projectId: string) => createInput(projectId), + rendered: (projectId: string) => ({ + ...createInput(projectId, '---\ntitle: Architecture\n---\n\nShared page'), + name: 'Architecture', + contentType: 'text/x-sim-page', + }), } + for (const [read, input] of Object.entries(shared)) + for (const change of ['disabled', 'password', 'archived', 'provenance'] as const) { + const f = await fixture() + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: input(f.projectId), + }) + const { share } = await updateProjectFileShare.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: file.id, + isActive: true, + authType: 'password', + password: 'initial-password', + }, + }) + const access = await authorizePublicFileShare({ + token: share.token, + credential: { method: 'POST', password: 'initial-password' }, + }) + if (!access.authorized) throw new Error('Initial credential rejected') + const applyChange = async () => { + if (change === 'disabled') + await db + .update(publicShare) + .set({ isActive: false }) + .where(eq(publicShare.id, share.id)) + if (change === 'password') + await db + .update(publicShare) + .set({ password: (await encryptSecret('replacement-password')).encrypted }) + .where(eq(publicShare.id, share.id)) + if (change === 'provenance') + await db + .update(workspaceFileSecretProvenance) + .set({ status: 'unknown' }) + .where(eq(workspaceFileSecretProvenance.fileId, file.id)) + if (change === 'archived') + await db + .update(workspaceFiles) + .set({ deletedAt: new Date() }) + .where(eq(workspaceFiles.id, file.id)) + } + // The change lands while the object is being read, buffered or streamed alike. + const download = storage.downloadFile + const downloadStream = storage.downloadFileStream + const spies = [ + vi.spyOn(storage, 'downloadFile').mockImplementation(async (options) => { + const result = await download(options) + if (options.key === file.key) await applyChange() + return result + }), + vi.spyOn(storage, 'downloadFileStream').mockImplementation(async (options) => { + const result = await downloadStream(options) + if (options.key === file.key) await applyChange() + return result + }), + ] + const delivered = readPublicFileShareContent({ grant: access.grant }) + await expect(sharedBytes(delivered)).rejects.toMatchObject({ + code: change === 'provenance' ? 'forbidden' : 'not_found', + }) + // Each fixture took the path it names: the stream refuses on open, the page as it renders. + if (read === 'streamed') expect(await delivered).toMatchObject({ kind: 'stream' }) + else await expect(delivered).rejects.toBeDefined() + for (const spy of spies) spy.mockRestore() + } } ) diff --git a/apps/sim/lib/public-shares/application/read.ts b/apps/sim/lib/public-shares/application/read.ts index 9248e77adac..cd35d316e83 100644 --- a/apps/sim/lib/public-shares/application/read.ts +++ b/apps/sim/lib/public-shares/application/read.ts @@ -34,11 +34,7 @@ import { import { storedObjectEtag } from '@/lib/uploads/server/delivery' import { hasEmbeddedFileRef } from '@/lib/uploads/server/embedded-image-refs' import { resolveServableImageBytes } from '@/lib/uploads/server/image-derivative' -import { - type ByteRange, - getWorkspaceFileSize, - MAX_BUFFERED_TRANSFER_BYTES, -} from '@/lib/uploads/shared/types' +import { type ByteRange, MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { resolveEffectiveMimeType } from '@/lib/uploads/utils/file' import { sniffImageContentType } from '@/lib/uploads/utils/validation' import { reportWorkspaceFileDelivery } from '@/lib/workspace-files/application/file-delivery-observer' @@ -143,6 +139,8 @@ function recordPublicShareDownload(input: { servedFile: WorkspaceFileRow operation: PublicFileReadOperation bytes: number + /** The inclusive span a ranged response carried; absent for the whole file. */ + range?: ByteRange request?: OrchestrationRequestContext }) { const { servedFile } = input @@ -164,6 +162,7 @@ function recordPublicShareDownload(input: { anonymous: true, owner: input.owner, bytes: input.bytes, + ...(input.range ? { range: `${input.range.start}-${input.range.end}` } : {}), }, }) } @@ -299,30 +298,34 @@ export async function checkPublicFileShareContent( /** * A shared file served as its stored bytes, read from storage only as far as a response needs: * the whole object, or one inclusive span. The bytes are never held, so a file of any size is - * served and a byte range is read on its own. + * served and a byte range is read on its own. Storage is asked for the object first, so a missing + * object is refused before any header is sent, and every length is measured against the object. * * `open` rechecks the grant, the revision, and the file's provenance once storage has opened and - * before the stream is handed out, as a buffered read does after rendering. One view is audited - * once: a download, or the first span of a player's ranged reads. + * before the stream is handed out, as a buffered read does after rendering. Each delivered + * response is reported and audited, with the span it carried. */ -function streamSharedFile( +async function streamSharedFile( grant: VerifiedPublicFileShareGrant, initial: PublicFileShareSnapshot, operation: PublicFileReadOperation, request?: OrchestrationRequestContext ) { const { file } = initial - const size = getWorkspaceFileSize(file) + const context = initial.adapter.storageContext + const stored = await headObject(file.key, context) + if (!stored) throw new OrchestrationError('not_found', 'File not found') + const { size } = stored return { kind: 'stream' as const, filename: file.originalName, contentType: resolveEffectiveMimeType(undefined, file.originalName), size, /** A content write uploads under a fresh key and stamps `contentUpdatedAt`. */ - etag: storedObjectEtag([file.key, size, file.contentUpdatedAt.getTime()]), + etag: storedObjectEtag([file.key, size, file.contentUpdatedAt.getTime(), stored.version ?? '']), async open(range: ByteRange | undefined): Promise> { const stream = nodeReadableToWebStream( - await downloadFileStream({ key: file.key, context: initial.adapter.storageContext, range }) + await downloadFileStream({ key: file.key, context, range }) ) try { const provenance = await withPublicFileShareGrant( @@ -338,18 +341,17 @@ function streamSharedFile( await stream.cancel(error).catch(() => undefined) throw asOrchestrationError(error) ?? error } - if (!range || range.start === 0) { - recordPublicShareDownload({ - workspaceId: initial.workspaceId, - organizationId: initial.organizationId, - owner: initial.owner, - sharedFile: file, - servedFile: file, - operation, - bytes: range ? range.end - range.start + 1 : size, - request, - }) - } + recordPublicShareDownload({ + workspaceId: initial.workspaceId, + organizationId: initial.organizationId, + owner: initial.owner, + sharedFile: file, + servedFile: file, + operation, + bytes: range ? range.end - range.start + 1 : size, + range, + request, + }) return stream }, } @@ -374,7 +376,7 @@ export async function readPublicFileShareContent({ return snapshot }) if (!preview && servesStoredBytes(initial.file.contentType, initial.file.originalName)) { - return streamSharedFile(grant, initial, operation, request) + return await streamSharedFile(grant, initial, operation, request) } const raw = await downloadFile({ key: initial.file.key, diff --git a/apps/sim/scripts/test-project-file-public-e2e.ts b/apps/sim/scripts/test-project-file-public-e2e.ts index 12db759cc67..fb67a147adb 100644 --- a/apps/sim/scripts/test-project-file-public-e2e.ts +++ b/apps/sim/scripts/test-project-file-public-e2e.ts @@ -60,12 +60,18 @@ const image = await sharp({ async function request( path: string, - options: { method?: string; body?: Record; cookie?: string } = {} + options: { + method?: string + body?: Record + cookie?: string + headers?: Record + } = {} ) { const headers = new Headers({ Origin: base.origin, 'Content-Type': 'application/json', 'X-Forwarded-For': proofIp, + ...options.headers, }) if (options.cookie) headers.set('Cookie', options.cookie) // boundary-raw-fetch: this proof exercises actual public binary, RSC, and credential HTTP surfaces. @@ -281,10 +287,13 @@ try { ) }) await check( - `${owner}: source and inline scan enforce independent real byte ceilings`, + `${owner}: stored bytes stream past the buffered ceiling while the inline scan keeps its own`, async () => { - await withSparseFile(required(doc.id), 100 * 1024 * 1024 + 1, async () => { - assertBinaryError(await request(`${path}/content`), 413) + const size = 100 * 1024 * 1024 + 1 + await withSparseFile(required(doc.id), size, async () => { + const head = await request(`${path}/content`, { headers: { range: 'bytes=0-0' } }) + assert.equal(head.status, 206) + assert.equal(head.headers.get('content-range'), `bytes 0-0/${size}`) assertBinaryError(await request(`${path}/inline?fileId=${required(photo.id)}`), 404) }) await withSparseFile(required(photo.id), 100 * 1024 * 1024 + 1, async () => {