diff --git a/apps/sim/app/api/files/public/[token]/content/route.ts b/apps/sim/app/api/files/public/[token]/content/route.ts index b85fe6df606..e665b3a474c 100644 --- a/apps/sim/app/api/files/public/[token]/content/route.ts +++ b/apps/sim/app/api/files/public/[token]/content/route.ts @@ -17,7 +17,10 @@ import { } from '@/lib/public-shares/application' import { enforcePublicFileRateLimit } from '@/lib/public-shares/rate-limit' import { FILE_CACHE_CONTROL } from '@/lib/uploads/server/delivery' -import { createConditionalFileResponse } from '@/app/api/files/utils' +import { + createConditionalFileResponse, + createStoredFileStreamResponse, +} from '@/app/api/files/utils' export const dynamic = 'force-dynamic' const logger = createLogger('PublicFileContentAPI') @@ -49,6 +52,12 @@ export const GET = withRouteHandler( preview: parsed.data.query.preview === '1', request, }) + if (result.kind === 'stream') { + return await createStoredFileStreamResponse( + { ...result, cacheControl: FILE_CACHE_CONTROL.revalidate }, + request.headers + ) + } return createConditionalFileResponse( { buffer: result.buffer, diff --git a/apps/sim/hooks/use-file-content-source.tsx b/apps/sim/hooks/use-file-content-source.tsx index 9dd838cc9b6..803e54dcee3 100644 --- a/apps/sim/hooks/use-file-content-source.tsx +++ b/apps/sim/hooks/use-file-content-source.tsx @@ -146,12 +146,17 @@ export function createPublicFileContentSource( contentUrl: string, owner?: EditableFileOwner ): FileContentSource { - return inlineImageSource( - (_key, opts) => - opts?.preview ? `${contentUrl}${contentUrl.includes('?') ? '&' : '?'}preview=1` : contentUrl, - `/api/files/public/${token}/inline`, - owner - ) + return { + ...inlineImageSource( + (_key, opts) => + opts?.preview + ? `${contentUrl}${contentUrl.includes('?') ? '&' : '?'}preview=1` + : contentUrl, + `/api/files/public/${token}/inline`, + owner + ), + servesByteRanges: true, + } } /** diff --git a/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts b/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts index 4dc2fbaf745..841573b0f67 100644 --- a/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts +++ b/apps/sim/lib/projects/files/__integration__/content-sharing.integration.ts @@ -1,4 +1,4 @@ -import { mkdtempSync } from 'node:fs' +import { existsSync, mkdtempSync } from 'node:fs' import { mkdir, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' @@ -39,8 +39,11 @@ import { createProjectFile, updateProjectFileContent, } from '@/lib/projects/files/application/content' +import type { readPublicFileShareContent } from '@/lib/public-shares/application' import * as shareManager from '@/lib/public-shares/manager' import * as storage from '@/lib/uploads/core/storage-service' +import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { sanitizeFileKey } from '@/lib/uploads/utils/file' vi.mock('@/lib/core/config/feature-flags', () => featureFlagsMock) @@ -129,6 +132,16 @@ async function fixture() { return { ...ids, principal: createSessionPrincipal({ userId: editorId }) } } +type PublicShareContent = Awaited> + +/** The bytes a public read delivers, whether streamed from storage or built whole. */ +async function sharedBytes(read: PublicShareContent | Promise) { + const result = await read + return result.kind === 'stream' + ? Buffer.from(await new Response(await result.open(undefined)).arrayBuffer()) + : result.buffer +} + function createInput(projectId: string, content = 'Project architecture') { return { projectId, @@ -410,9 +423,9 @@ describe('Public file shares against PostgreSQL and private storage', () => { credential: { method: 'POST', password: 'correct-password' }, }) if (!accepted.authorized) throw new Error('Correct share password rejected') - expect((await readPublicFileShareContent({ grant: accepted.grant })).buffer.toString()).toBe( - 'Shared architecture' - ) + expect( + (await sharedBytes(readPublicFileShareContent({ grant: accepted.grant }))).toString() + ).toBe('Shared architecture') const [passwordShare] = await db .select() .from(publicShare) @@ -653,6 +666,74 @@ describe('Public file shares against PostgreSQL and private storage', () => { } ) + check( + 'public share streams stored bytes past the buffered ceiling and refuses once the share is disabled', + async () => { + const { updateProjectFileShare } = await import('@/lib/projects/files/application/shares') + const { authorizePublicFileShare, readPublicFileShareContent } = await import( + '@/lib/public-shares/application' + ) + const f = await fixture() + const share = async (name: string, contentType: string) => { + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: { ...createInput(f.projectId, 'placeholder'), name, contentType }, + }) + const { share } = await updateProjectFileShare.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: file.id, isActive: true }, + }) + const access = await authorizePublicFileShare({ + token: share.token, + credential: { method: 'GET' }, + }) + if (!access.authorized) throw new Error(`Public ${name} fixture rejected`) + return { file, grant: access.grant } + } + + const recording = await share('recording.mp4', 'video/mp4') + const size = MAX_BUFFERED_TRANSFER_BYTES + 1024 + const bytes = Buffer.alloc(size) + bytes.write('tail', size - 4) + const [row] = await db + .select() + .from(workspaceFiles) + .where(eq(workspaceFiles.id, recording.file.id)) + const objectPath = join(localStorageRoot, sanitizeFileKey(row.key)) + expect(existsSync(objectPath)).toBe(true) + await writeFile(objectPath, bytes) + await db + .update(workspaceFiles) + .set({ sizeBytes: size }) + .where(eq(workspaceFiles.id, recording.file.id)) + + const stored = await readPublicFileShareContent({ grant: recording.grant }) + if (stored.kind !== 'stream') throw new Error('A stored recording was buffered') + expect(stored.size).toBe(size) + const whole = Buffer.from(await new Response(await stored.open(undefined)).arrayBuffer()) + expect(whole.length).toBe(size) + expect(whole.subarray(size - 4).toString()).toBe('tail') + const span = await stored.open({ start: size - 4, end: size - 1 }) + expect(Buffer.from(await new Response(span).arrayBuffer()).toString()).toBe('tail') + + await updateProjectFileShare.execute({ + principal: f.principal, + input: { projectId: f.projectId, fileId: recording.file.id, isActive: false }, + }) + await expect(stored.open({ start: 0, end: 3 })).rejects.toMatchObject({ code: 'not_found' }) + + const page = await share('notes.html', 'text/html') + expect(await readPublicFileShareContent({ grant: page.grant })).toMatchObject({ + kind: 'buffer', + }) + // A generation source is compiled, never streamed: with no cached artifact it refuses. + const source = await share('report.pptx', 'text/x-pptxgenjs') + await expect(readPublicFileShareContent({ grant: source.grant })).rejects.toMatchObject({ + code: 'conflict', + }) + } + ) + check( 'public share Office artifacts require the current owner-qualified dependency cache without compilation', async () => { @@ -693,7 +774,7 @@ describe('Public file shares against PostgreSQL and private storage', () => { principal: f.principal, input: { projectId: f.projectId, fileId: source.file.id, maxBytes: 1024 }, }) - expect((await readPublicFileShareContent({ grant: access.grant })).buffer).toEqual( + expect(await sharedBytes(readPublicFileShareContent({ grant: access.grant }))).toEqual( rendered.buffer ) await updateProjectFileContent.execute({ @@ -719,31 +800,38 @@ describe('Public file shares against PostgreSQL and private storage', () => { '@/lib/public-shares/application' ) const { encryptSecret } = await import('@/lib/core/security/encryption') - for (const change of ['disabled', 'password', 'archived', 'provenance'] as const) { - const f = await fixture() - const { file } = await createProjectFile.execute({ - principal: f.principal, - input: createInput(f.projectId), - }) - const { share } = await updateProjectFileShare.execute({ - principal: f.principal, - input: { - projectId: f.projectId, - fileId: file.id, - isActive: true, - authType: 'password', - password: 'initial-password', - }, - }) - const access = await authorizePublicFileShare({ - token: share.token, - credential: { method: 'POST', password: 'initial-password' }, - }) - if (!access.authorized) throw new Error('Initial credential rejected') - const download = storage.downloadFile - const spy = vi.spyOn(storage, 'downloadFile').mockImplementation(async (options) => { - const result = await download(options) - if (options.key === file.key) { + // A Markdown file streams from storage; a page is rendered whole. Both reads must refuse. + const shared = { + streamed: (projectId: string) => createInput(projectId), + rendered: (projectId: string) => ({ + ...createInput(projectId, '---\ntitle: Architecture\n---\n\nShared page'), + name: 'Architecture', + contentType: 'text/x-sim-page', + }), + } + for (const [read, input] of Object.entries(shared)) + for (const change of ['disabled', 'password', 'archived', 'provenance'] as const) { + const f = await fixture() + const { file } = await createProjectFile.execute({ + principal: f.principal, + input: input(f.projectId), + }) + const { share } = await updateProjectFileShare.execute({ + principal: f.principal, + input: { + projectId: f.projectId, + fileId: file.id, + isActive: true, + authType: 'password', + password: 'initial-password', + }, + }) + const access = await authorizePublicFileShare({ + token: share.token, + credential: { method: 'POST', password: 'initial-password' }, + }) + if (!access.authorized) throw new Error('Initial credential rejected') + const applyChange = async () => { if (change === 'disabled') await db .update(publicShare) @@ -765,13 +853,30 @@ describe('Public file shares against PostgreSQL and private storage', () => { .set({ deletedAt: new Date() }) .where(eq(workspaceFiles.id, file.id)) } - return result - }) - await expect(readPublicFileShareContent({ grant: access.grant })).rejects.toMatchObject({ - code: change === 'provenance' ? 'forbidden' : 'not_found', - }) - spy.mockRestore() - } + // The change lands while the object is being read, buffered or streamed alike. + const download = storage.downloadFile + const downloadStream = storage.downloadFileStream + const spies = [ + vi.spyOn(storage, 'downloadFile').mockImplementation(async (options) => { + const result = await download(options) + if (options.key === file.key) await applyChange() + return result + }), + vi.spyOn(storage, 'downloadFileStream').mockImplementation(async (options) => { + const result = await downloadStream(options) + if (options.key === file.key) await applyChange() + return result + }), + ] + const delivered = readPublicFileShareContent({ grant: access.grant }) + await expect(sharedBytes(delivered)).rejects.toMatchObject({ + code: change === 'provenance' ? 'forbidden' : 'not_found', + }) + // Each fixture took the path it names: the stream refuses on open, the page as it renders. + if (read === 'streamed') expect(await delivered).toMatchObject({ kind: 'stream' }) + else await expect(delivered).rejects.toBeDefined() + for (const spy of spies) spy.mockRestore() + } } ) @@ -836,7 +941,7 @@ describe('Public file shares against PostgreSQL and private storage', () => { }) if (!legacyAccess.authorized) throw new Error('Existing workspace share rejected') expect( - (await readPublicFileShareContent({ grant: legacyAccess.grant })).buffer.toString() + (await sharedBytes(readPublicFileShareContent({ grant: legacyAccess.grant }))).toString() ).toBe('Workspace link') } ) @@ -892,7 +997,7 @@ describe('Public file shares against PostgreSQL and private storage', () => { if (!access.authorized) throw new Error('Public page rejected') const rendered = await readPublicFileShareContent({ grant: access.grant }) expect(rendered.contentType).toBe('text/html') - expect(rendered.buffer.toString()).toContain( + expect((await sharedBytes(rendered)).toString()).toContain( `data:image/png;base64,${image.toString('base64')}` ) expect( diff --git a/apps/sim/lib/public-shares/application/read.ts b/apps/sim/lib/public-shares/application/read.ts index 9b881e76820..cd35d316e83 100644 --- a/apps/sim/lib/public-shares/application/read.ts +++ b/apps/sim/lib/public-shares/application/read.ts @@ -7,6 +7,7 @@ import { OrchestrationError, type OrchestrationRequestContext, } from '@/lib/core/orchestration/types' +import { nodeReadableToWebStream } from '@/lib/core/utils/node-stream' import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import type { DbTransaction } from '@/lib/db/types' import { @@ -23,16 +24,17 @@ import { mergeWorkspaceFileSecretProvenance, type WorkspaceFileSecretProvenance, } from '@/lib/uploads/contexts/workspace/file-secret-provenance' -import { downloadFile, headObject } from '@/lib/uploads/core/storage-service' +import { downloadFile, downloadFileStream, headObject } from '@/lib/uploads/core/storage-service' import { collectReferencedFileIds, getDocumentSourceLanguage, getE2BDocFormat, isCompiledDocumentBuffer, } from '@/lib/uploads/documents/compile' +import { storedObjectEtag } from '@/lib/uploads/server/delivery' import { hasEmbeddedFileRef } from '@/lib/uploads/server/embedded-image-refs' import { resolveServableImageBytes } from '@/lib/uploads/server/image-derivative' -import { MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' +import { type ByteRange, MAX_BUFFERED_TRANSFER_BYTES } from '@/lib/uploads/shared/types' import { resolveEffectiveMimeType } from '@/lib/uploads/utils/file' import { sniffImageContentType } from '@/lib/uploads/utils/validation' import { reportWorkspaceFileDelivery } from '@/lib/workspace-files/application/file-delivery-observer' @@ -44,6 +46,7 @@ import { collectSimPageFileReferences, inlineSimPageImages, } from '@/lib/workspace-files/page-document.server' +import { servesStoredBytes } from '@/lib/workspace-files/stored-bytes' interface PublicReadInput { grant: VerifiedPublicFileShareGrant @@ -127,6 +130,42 @@ async function dependencies( await admitPublicFiles(tx, rows) return rows } +/** Audits one anonymous delivery of a shared file, or of a document input it embedded. */ +function recordPublicShareDownload(input: { + workspaceId: string | null + organizationId: string | null + owner: PublicFileShareSnapshot['owner'] + sharedFile: WorkspaceFileRow + servedFile: WorkspaceFileRow + operation: PublicFileReadOperation + bytes: number + /** The inclusive span a ranged response carried; absent for the whole file. */ + range?: ByteRange + request?: OrchestrationRequestContext +}) { + const { servedFile } = input + recordAudit({ + actorId: null, + workspaceId: input.workspaceId, + action: AuditAction.FILE_DOWNLOADED, + resourceType: AuditResourceType.FILE, + resourceId: servedFile.id, + resourceName: servedFile.originalName, + description: `Public share download of "${servedFile.originalName}"`, + request: input.request, + metadata: { + organizationId: input.organizationId, + operation: input.operation.id, + sharedDocumentId: input.sharedFile.id, + sharedByUserId: input.sharedFile.userId, + access: 'public_share', + anonymous: true, + owner: input.owner, + bytes: input.bytes, + ...(input.range ? { range: `${input.range.start}-${input.range.end}` } : {}), + }, + }) +} async function finish( grant: VerifiedPublicFileShareGrant, manifest: PublicManifest, @@ -165,25 +204,15 @@ async function finish( } }) await reportWorkspaceFileDelivery(result.secretProvenance, servedFile.id) - recordAudit({ - actorId: null, + recordPublicShareDownload({ workspaceId: manifest.snapshot.workspaceId, - action: AuditAction.FILE_DOWNLOADED, - resourceType: AuditResourceType.FILE, - resourceId: servedFile.id, - resourceName: servedFile.originalName, - description: `Public share download of "${servedFile.originalName}"`, + organizationId: result.organizationId, + owner: result.owner, + sharedFile: manifest.snapshot.file, + servedFile, + operation, + bytes: buffer.length, request, - metadata: { - organizationId: result.organizationId, - operation: operation.id, - sharedDocumentId: result.file.id, - sharedByUserId: manifest.snapshot.file.userId, - access: 'public_share', - anonymous: true, - owner: result.owner, - bytes: buffer.length, - }, }) return result } @@ -207,8 +236,10 @@ export async function checkPublicFileShareContent( }) const stored = await headObject(initial.file.key, initial.adapter.storageContext) if (!stored) throw new OrchestrationError('not_found', 'File not found') - assertKnownSizeWithinLimit(stored.size, MAX_BUFFERED_TRANSFER_BYTES, 'File') - const format = await getE2BDocFormat(initial.file.originalName) + // Stored bytes stream at any size; only a representation built from them is held whole. + const streamed = servesStoredBytes(initial.file.contentType, initial.file.originalName) + if (!streamed) assertKnownSizeWithinLimit(stored.size, MAX_BUFFERED_TRANSFER_BYTES, 'File') + const format = streamed ? null : await getE2BDocFormat(initial.file.originalName) let inputs: WorkspaceFileRow[] = [] if (format) { const source = await downloadFile({ @@ -264,7 +295,75 @@ export async function checkPublicFileShareContent( } } -/** A public read never executes generated document code; it serves current cached artifacts only. */ +/** + * A shared file served as its stored bytes, read from storage only as far as a response needs: + * the whole object, or one inclusive span. The bytes are never held, so a file of any size is + * served and a byte range is read on its own. Storage is asked for the object first, so a missing + * object is refused before any header is sent, and every length is measured against the object. + * + * `open` rechecks the grant, the revision, and the file's provenance once storage has opened and + * before the stream is handed out, as a buffered read does after rendering. Each delivered + * response is reported and audited, with the span it carried. + */ +async function streamSharedFile( + grant: VerifiedPublicFileShareGrant, + initial: PublicFileShareSnapshot, + operation: PublicFileReadOperation, + request?: OrchestrationRequestContext +) { + const { file } = initial + const context = initial.adapter.storageContext + const stored = await headObject(file.key, context) + if (!stored) throw new OrchestrationError('not_found', 'File not found') + const { size } = stored + return { + kind: 'stream' as const, + filename: file.originalName, + contentType: resolveEffectiveMimeType(undefined, file.originalName), + size, + /** A content write uploads under a fresh key and stamps `contentUpdatedAt`. */ + etag: storedObjectEtag([file.key, size, file.contentUpdatedAt.getTime(), stored.version ?? '']), + async open(range: ByteRange | undefined): Promise> { + const stream = nodeReadableToWebStream( + await downloadFileStream({ key: file.key, context, range }) + ) + try { + const provenance = await withPublicFileShareGrant( + grant, + operation, + async (tx, snapshot) => { + requireRevision(snapshot.file, file) + return admitPublicFiles(tx, [snapshot.file]) + } + ) + await reportWorkspaceFileDelivery(provenance, file.id) + } catch (error) { + await stream.cancel(error).catch(() => undefined) + throw asOrchestrationError(error) ?? error + } + recordPublicShareDownload({ + workspaceId: initial.workspaceId, + organizationId: initial.organizationId, + owner: initial.owner, + sharedFile: file, + servedFile: file, + operation, + bytes: range ? range.end - range.start + 1 : size, + range, + request, + }) + return stream + }, + } +} + +/** + * Reads a shared file for delivery. A file served as its stored bytes comes back as a stream of any + * size (see {@link servesStoredBytes}); a page or generated document is rendered, and a `preview` + * may be a HEIC transcode sniffed from the bytes, so those are built whole under the buffered + * ceiling. A public read never executes generated document code; it serves current cached + * artifacts only. + */ export async function readPublicFileShareContent({ grant, preview, @@ -276,6 +375,9 @@ export async function readPublicFileShareContent({ await admitPublicFiles(tx, [snapshot.file]) return snapshot }) + if (!preview && servesStoredBytes(initial.file.contentType, initial.file.originalName)) { + return await streamSharedFile(grant, initial, operation, request) + } const raw = await downloadFile({ key: initial.file.key, context: initial.adapter.storageContext, @@ -361,7 +463,10 @@ export async function readPublicFileShareContent({ const image = await resolveServableImageBytes(raw, initial.file.key, { owner: initial.owner }) if (image) ({ buffer, contentType } = image) } - return await finish(grant, manifest, operation, buffer, contentType, request) + return { + kind: 'buffer' as const, + ...(await finish(grant, manifest, operation, buffer, contentType, request)), + } } catch (error) { throw asOrchestrationError(error) ?? error } diff --git a/apps/sim/scripts/test-project-file-public-e2e.ts b/apps/sim/scripts/test-project-file-public-e2e.ts index 12db759cc67..fb67a147adb 100644 --- a/apps/sim/scripts/test-project-file-public-e2e.ts +++ b/apps/sim/scripts/test-project-file-public-e2e.ts @@ -60,12 +60,18 @@ const image = await sharp({ async function request( path: string, - options: { method?: string; body?: Record; cookie?: string } = {} + options: { + method?: string + body?: Record + cookie?: string + headers?: Record + } = {} ) { const headers = new Headers({ Origin: base.origin, 'Content-Type': 'application/json', 'X-Forwarded-For': proofIp, + ...options.headers, }) if (options.cookie) headers.set('Cookie', options.cookie) // boundary-raw-fetch: this proof exercises actual public binary, RSC, and credential HTTP surfaces. @@ -281,10 +287,13 @@ try { ) }) await check( - `${owner}: source and inline scan enforce independent real byte ceilings`, + `${owner}: stored bytes stream past the buffered ceiling while the inline scan keeps its own`, async () => { - await withSparseFile(required(doc.id), 100 * 1024 * 1024 + 1, async () => { - assertBinaryError(await request(`${path}/content`), 413) + const size = 100 * 1024 * 1024 + 1 + await withSparseFile(required(doc.id), size, async () => { + const head = await request(`${path}/content`, { headers: { range: 'bytes=0-0' } }) + assert.equal(head.status, 206) + assert.equal(head.headers.get('content-range'), `bytes 0-0/${size}`) assertBinaryError(await request(`${path}/inline?fileId=${required(photo.id)}`), 404) }) await withSparseFile(required(photo.id), 100 * 1024 * 1024 + 1, async () => {