Skip to content

Commit a6d8ab9

Browse files
committed
fix(files): preserve owned content and generated search delivery
1 parent 6b4a58f commit a6d8ab9

11 files changed

Lines changed: 197 additions & 114 deletions

File tree

‎apps/sim/lib/uploads/contexts/workspace/__integration__/file-versions.integration.ts‎

Lines changed: 48 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
outboxEvent,
1111
permissions,
1212
user,
13+
workflowTest,
1314
workspace,
1415
workspaceFileSecretProvenance,
1516
workspaceFiles,
@@ -43,6 +44,7 @@ import { createAgentCliTransport } from '@/lib/mothership/agent-cli/transport'
4344
import { handoffFileCreatorsInTx } from '@/lib/uploads/contexts/workspace/creator-handoff'
4445
import { WORKSPACE_FILE_STORAGE_CLEANUP_OUTBOX_EVENT } from '@/lib/uploads/contexts/workspace/file-outbox-events'
4546
import {
47+
createOwnedWorkspaceFile,
4648
deleteWorkspaceFileVersion,
4749
fetchWorkspaceFileBuffer,
4850
getWorkspaceFile,
@@ -64,13 +66,7 @@ import {
6466
releaseWorkspaceFileVersionsForPurgeInTx,
6567
} from '@/lib/uploads/contexts/workspace/workspace-file-versions'
6668
import * as storageService from '@/lib/uploads/core/storage-service'
67-
import { v2FileErrorPolicies } from '@/lib/workspace-files/api'
68-
import { presentWorkspaceFileText } from '@/lib/workspace-files/api/text-presenter'
69-
import {
70-
downloadWorkspaceFileVersion,
71-
readWorkspaceFileVersionText,
72-
revertWorkspaceFileVersion,
73-
} from '@/lib/workspace-files/application/file-versions'
69+
import { revertWorkspaceFileVersion } from '@/lib/workspace-files/application/file-versions'
7470
import { revokeWorkspaceAccessTx } from '@/lib/workspaces/access/workspace-access'
7571
import { runCleanupFileVersions } from '@/background/cleanup-file-versions'
7672
import { runCleanupSoftDeletes } from '@/background/cleanup-soft-deletes'
@@ -355,6 +351,7 @@ describe('workspace file version history in PostgreSQL', () => {
355351
)
356352

357353
it.each([
354+
{ operation: 'owned', enqueueAvailable: true, code: 'CONNECTION_CLOSED' },
358355
{ operation: 'upload', enqueueAvailable: true, code: undefined },
359356
{ operation: 'content', enqueueAvailable: true, code: undefined },
360357
{ operation: 'upload', enqueueAvailable: false, code: undefined },
@@ -389,23 +386,41 @@ describe('workspace file version history in PostgreSQL', () => {
389386
})
390387
try {
391388
const result =
392-
operation === 'content'
393-
? updateWorkspaceFileContent(
394-
fixture.workspaceId,
395-
fixture.fileId,
396-
fixture.aliceId,
397-
Buffer.from('committed content'),
398-
undefined,
399-
{ version: { source: 'api', authorUserId: fixture.aliceId } }
400-
)
401-
: uploadWorkspaceFile(
402-
fixture.workspaceId,
403-
fixture.aliceId,
404-
Buffer.from('committed content'),
405-
'committed.txt',
406-
'text/plain',
407-
{ notifyWorkspaceChange: false }
408-
)
389+
operation === 'owned'
390+
? createOwnedWorkspaceFile({
391+
context: 'test',
392+
workspaceId: fixture.workspaceId,
393+
userId: fixture.aliceId,
394+
content: 'committed content',
395+
contentType: 'text/plain',
396+
fileName: (id) => `${id}.txt`,
397+
insertOwner: async (tx, bodyFileId) =>
398+
tx.insert(workflowTest).values({
399+
id: generateId(),
400+
workspaceId: fixture.workspaceId,
401+
name: generateId(),
402+
title: 'Commit outcome',
403+
bodyFileId,
404+
sourceHash: 'fixture',
405+
}),
406+
})
407+
: operation === 'content'
408+
? updateWorkspaceFileContent(
409+
fixture.workspaceId,
410+
fixture.fileId,
411+
fixture.aliceId,
412+
Buffer.from('committed content'),
413+
undefined,
414+
{ version: { source: 'api', authorUserId: fixture.aliceId } }
415+
)
416+
: uploadWorkspaceFile(
417+
fixture.workspaceId,
418+
fixture.aliceId,
419+
Buffer.from('committed content'),
420+
'committed.txt',
421+
'text/plain',
422+
{ notifyWorkspaceChange: false }
423+
)
409424
await expect(result).rejects.toThrow('Commit acknowledgement lost')
410425
} finally {
411426
capture.mockRestore()
@@ -417,11 +432,15 @@ describe('workspace file version history in PostgreSQL', () => {
417432
.from(workspaceFiles)
418433
.where(eq(workspaceFiles.key, stagedKey))
419434
expect(committed?.id).toEqual(expect.any(String))
420-
const file = await getWorkspaceFile(fixture.workspaceId, committed.id)
421-
if (!file) throw new Error('committed file missing')
422-
expect((await fetchWorkspaceFileBuffer(file, { maxBytes: 1024 })).toString()).toBe(
423-
'committed content'
424-
)
435+
expect(
436+
(
437+
await storageService.downloadFile({
438+
key: stagedKey,
439+
context: 'workspace',
440+
maxBytes: 1024,
441+
})
442+
).toString()
443+
).toBe('committed content')
425444
expect(
426445
await db
427446
.select({ id: outboxEvent.id })

‎apps/sim/lib/uploads/contexts/workspace/workspace-file-manager.ts‎

Lines changed: 24 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -789,54 +789,36 @@ export async function createOwnedWorkspaceFile<T>(params: {
789789
const fileId = `wf_${generateShortId()}`
790790
const name = params.fileName(fileId)
791791
const buffer = Buffer.from(params.content, 'utf-8')
792-
const storageKey = generateWorkspaceFileKey(params.workspaceId, name)
793792
const storageBillingContext = await resolveStorageBillingContext(params.workspaceId)
794-
const uploadResult = await uploadFile({
795-
file: buffer,
796-
fileName: storageKey,
793+
const staged = await stageFileContent({
794+
owner: { entityType: 'workspace', entityId: params.workspaceId },
795+
userId: params.userId,
796+
name,
797797
contentType: params.contentType,
798-
context: 'workspace',
799-
preserveKey: true,
800-
customKey: storageKey,
801-
metadata: {
802-
originalName: name,
803-
uploadedAt: new Date().toISOString(),
804-
purpose: 'workspace',
798+
content: buffer,
799+
})
800+
const { owner, updatedUsage } = await finalizeStagedFileContent(staged, async (tx) => {
801+
const inserted = await insertWorkspaceFileMetadataInTx(tx, {
802+
id: fileId,
803+
key: staged.key,
805804
userId: params.userId,
806805
workspaceId: params.workspaceId,
807-
},
808-
persistMetadata: false,
809-
})
810-
try {
811-
const { owner, updatedUsage } = await db.transaction(async (tx) => {
812-
const inserted = await insertWorkspaceFileMetadataInTx(tx, {
813-
id: fileId,
814-
key: uploadResult.key,
815-
userId: params.userId,
816-
workspaceId: params.workspaceId,
817-
folderId: null,
818-
originalName: name,
819-
contentType: params.contentType,
820-
size: buffer.length,
821-
context: params.context,
822-
})
823-
if (!inserted) throw new Error(`Owned ${params.context} file ${fileId} was not inserted`)
824-
const updatedUsage = await incrementStorageUsageForBillingContextInTx(
825-
tx,
826-
storageBillingContext,
827-
buffer.length
828-
)
829-
return { owner: await params.insertOwner(tx, fileId), updatedUsage }
806+
folderId: null,
807+
originalName: name,
808+
contentType: params.contentType,
809+
size: buffer.length,
810+
context: params.context,
830811
})
831-
void maybeNotifyStorageLimitForBillingContext(storageBillingContext, updatedUsage)
832-
return { fileId, owner }
833-
} catch (error) {
834-
await cleanupWorkspaceStorageObject(
835-
uploadResult.key,
836-
`${params.context} file owner insert failure`
812+
if (!inserted) throw new Error(`Owned ${params.context} file ${fileId} was not inserted`)
813+
const updatedUsage = await incrementStorageUsageForBillingContextInTx(
814+
tx,
815+
storageBillingContext,
816+
buffer.length
837817
)
838-
throw error
839-
}
818+
return { owner: await params.insertOwner(tx, fileId), updatedUsage }
819+
})
820+
void maybeNotifyStorageLimitForBillingContext(storageBillingContext, updatedUsage)
821+
return { fileId, owner }
840822
}
841823

842824
/**

‎apps/sim/lib/uploads/documents/compile.ts‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,10 @@ import {
2929
iterateDocumentFileReferences,
3030
} from '@/lib/uploads/documents/references'
3131
import { getFileExtension, getMimeTypeFromExtension } from '@/lib/uploads/utils/file-utils'
32-
import { admitWorkspaceFiles } from '@/lib/workspace-files/application/admit-file-content'
32+
import {
33+
admitWorkspaceFiles,
34+
workspaceFileProvenanceIdentity,
35+
} from '@/lib/workspace-files/application/admit-file-content'
3336
import { readWorkspaceFileContent } from '@/lib/workspace-files/application/read-workspace-file-content'
3437
import { readWorkspaceFileMetadata } from '@/lib/workspace-files/application/read-workspace-file-metadata'
3538
import type { SandboxTaskId } from '@/sandbox-tasks/registry'
@@ -193,12 +196,12 @@ function touchesReferencedFiles(
193196
function referencedImageIdentities(
194197
resolution: ReferencedImageResolution
195198
): WorkspaceFileSecretProvenanceIdentity[] {
196-
return resolution.images.map(({ record }) => ({
197-
fileId: record.id,
198-
key: record.key,
199-
context: record.storageContext ?? 'workspace',
200-
contentUpdatedAt: record.contentUpdatedAt ?? record.updatedAt,
201-
}))
199+
return resolution.images.map(({ record }) =>
200+
workspaceFileProvenanceIdentity({
201+
...record,
202+
contentUpdatedAt: record.contentUpdatedAt ?? record.updatedAt,
203+
})
204+
)
202205
}
203206

204207
/** Collects static input identities, retaining one over the cap so callers can reject oversized sets. */

‎apps/sim/lib/uploads/documents/compiled-store.ts‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -164,8 +164,7 @@ export async function publishCompiledDocArtifact(
164164
owner: string | EditableFileOwner,
165165
source: string,
166166
ext: string,
167-
referencedInputIdentity: string,
168-
onArtifactWrite?: (key: string) => void
167+
referencedInputIdentity: string
169168
): Promise<void> {
170169
if (!referencedInputIdentity) {
171170
throw new Error('Published compiled document identity must not be empty')
@@ -175,7 +174,6 @@ export async function publishCompiledDocArtifact(
175174
if (existing?.referencedInputIdentity === referencedInputIdentity) return
176175
const pointer: PublishedArtifactPointer = { version: 1, referencedInputIdentity }
177176
try {
178-
onArtifactWrite?.(key)
179177
await uploadFile({
180178
file: Buffer.from(JSON.stringify(pointer), 'utf-8'),
181179
fileName: `doc.${ext}.published.json`,
@@ -285,12 +283,10 @@ export async function storeCompiledDoc(
285283
ext: string,
286284
contentType: string,
287285
binary: Buffer,
288-
referencedInputIdentity?: string,
289-
onArtifactWrite?: (key: string) => void
286+
referencedInputIdentity?: string
290287
): Promise<void> {
291288
const key = compiledArtifactKey(owner, source, ext, referencedInputIdentity)
292289
try {
293-
onArtifactWrite?.(key)
294290
await uploadFile({
295291
file: binary,
296292
fileName: `doc.${ext}`,
@@ -300,7 +296,7 @@ export async function storeCompiledDoc(
300296
preserveKey: true,
301297
})
302298
if (referencedInputIdentity) {
303-
await publishCompiledDocArtifact(owner, source, ext, referencedInputIdentity, onArtifactWrite)
299+
await publishCompiledDocArtifact(owner, source, ext, referencedInputIdentity)
304300
}
305301
} catch (err) {
306302
logger.error('Failed to store compiled doc artifact', {

‎apps/sim/lib/workspace-files/application/admit-file-content.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ export async function requireAdmittedContributors(
113113
* Admits each of many files on its own, classifying all of them in one batch, and returns the ids
114114
* admitted. A caller no gate observes gets every id back without a provenance read.
115115
*/
116-
export async function admitWorkspaceFileIdentities(
116+
async function admitWorkspaceFileIdentities(
117117
principal: Principal,
118118
workspaceId: string,
119119
identities: readonly WorkspaceFileSecretProvenanceIdentity[],

‎apps/sim/lib/workspace-files/application/fetch-servable-workspace-file-buffer.ts‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,12 @@ export async function fetchAuthorizedServableWorkspaceFileBuffer(
3939
}
4040
)
4141
const artifactKey = result.artifactKey
42-
if (artifactKey && result.contributingFiles?.every((file) => file.contentUpdatedAt) !== false) {
42+
if (
43+
artifactKey &&
44+
result.contributingFiles?.every(
45+
(file) => file.contentUpdatedAt && ['workspace', 'test', 'changelog'].includes(file.context)
46+
) !== false
47+
) {
4348
await db.transaction((tx) =>
4449
markFileSearchArtifactReadyInTx(tx, {
4550
owner: { entityType: 'workspace', entityId: fileRecord.workspaceId },
@@ -49,7 +54,7 @@ export async function fetchAuthorizedServableWorkspaceFileBuffer(
4954
contentUpdatedAt: fileRecord.contentUpdatedAt ?? fileRecord.updatedAt,
5055
},
5156
dependencies: (result.contributingFiles ?? []).map((file) => {
52-
if (!file.contentUpdatedAt || file.context !== 'workspace')
57+
if (!file.contentUpdatedAt)
5358
throw new Error('Artifact input has no canonical workspace revision')
5459
return {
5560
fileId: file.fileId,

‎apps/sim/lib/workspace-files/application/search-workspace-file-content.test.ts‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,13 @@ describe('searchWorkspaceFileContent cancellation', () => {
8383
})
8484
)
8585
await expect(
86-
searchWorkspaceFileContent.execute({
87-
principal,
88-
input,
89-
request: { headers: new Headers() },
90-
})
86+
observeContentDelivery(cleanOnlyGate, () =>
87+
searchWorkspaceFileContent.execute({
88+
principal: copilotPrincipal(),
89+
input,
90+
request: { headers: new Headers() },
91+
})
92+
)
9193
).rejects.toMatchObject({ code: 'locked' })
9294
})
9395
})
@@ -215,7 +217,18 @@ describe('searchWorkspaceFileContent for Chat', () => {
215217
expect(found.results[0].text).toMatch(/preview withheld/i)
216218
})
217219

218-
it("leaves a person's search limit and result untouched", async () => {
220+
it("preserves a person's point-in-time results after matching rows change", async () => {
221+
resetDbChainMock()
222+
queueTableRows(schemaMock.workspace, [
223+
{
224+
workspaceId: 'workspace-1',
225+
workspaceOrganizationId: null,
226+
allowPersonalApiKeys: true,
227+
billedAccountUserId: 'user-1',
228+
},
229+
])
230+
queueTableRows(schemaMock.workspaceFiles, [])
231+
queueTableRows(schemaMock.workspaceFileSearchRevision, [])
219232
const raw = {
220233
results: [hit('refused', 1)],
221234
count: 1,

‎apps/sim/lib/workspace-files/application/search-workspace-file-content.ts‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,22 @@ export const searchWorkspaceFileContent = defineAuthorizedWorkspaceFileUseCase({
8686
folderScope,
8787
signal,
8888
})
89+
if (!classified) {
90+
const ownedInputs = new Map(
91+
result.sources.flatMap((source) =>
92+
source.dependencies
93+
.filter((dependency) => dependency.context !== 'workspace')
94+
.map((dependency) => [dependency.fileId, dependency] as const)
95+
)
96+
)
97+
for (const dependency of ownedInputs.values())
98+
await assertOwnedFileAccess(principal, {
99+
...context,
100+
fileId: dependency.fileId,
101+
fileContext: dependency.context ?? '',
102+
})
103+
return toWorkspaceFileSearchResult(result)
104+
}
89105
const provenance = await db.transaction(async (tx) => {
90106
await lockWorkspaceProject(tx, context.workspaceId)
91107
const [current] = await tx
@@ -125,7 +141,6 @@ export const searchWorkspaceFileContent = defineAuthorizedWorkspaceFileUseCase({
125141
view: 'excerpt' as const,
126142
}))
127143
)
128-
if (!classified) return toWorkspaceFileSearchResult(result)
129144
const admittedIds = new Set(admitted.map((source) => source.identity.fileId))
130145
return toWorkspaceFileSearchResult({
131146
...result,

‎apps/sim/lib/workspace-files/search/dispatcher.integration.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,8 +119,10 @@ describe('workspace file search dispatch PostgreSQL deadlines', () => {
119119
await connection`CREATE TABLE workspace_files (
120120
id text PRIMARY KEY, workspace_id text, context text NOT NULL,
121121
deleted_at timestamp, content_updated_at timestamp NOT NULL,
122-
project_id text, key text NOT NULL DEFAULT 'key'
122+
project_id text, organization_id text, chat_id text, key text NOT NULL DEFAULT 'key'
123123
)`
124+
await connection`CREATE TABLE workflow_test (body_file_id text, workspace_id text, deleted_at timestamp)`
125+
await connection`CREATE TABLE changelog_release (body_file_id text, workspace_id text)`
124126
await connection`CREATE TABLE workspace_file_search_revision (
125127
file_id text PRIMARY KEY, workspace_id text NOT NULL,
126128
source_content_updated_at timestamp NOT NULL, status text NOT NULL DEFAULT 'pending',

0 commit comments

Comments
 (0)