Skip to content

Commit b6c80de

Browse files
waleedlatif1waleedlatif1
committed
perf(db): index live workspace files by folder upload date and name trigrams
Adds two partial indexes on the live workspace-file predicate, built concurrently: - workspace_files_workspace_folder_uploaded_idx on (workspace_id, coalesce(folder_id, ''), uploaded_at, id) serves a folder page by upload date without a sort and seeks a folder's files for its rollup. size_bytes stays out of the key to keep it narrow. - workspace_files_workspace_name_trgm_idx, a (workspace_id, original_name gin_trgm_ops) GIN with fastupdate off, like the other GIN indexes on hot-write tables, serves the case-insensitive name search. The integration test proves the plans on the exact SQL the managers send, including a workspace-wide page by upload date streaming workspace_files_workspace_active_uploaded_idx in both directions.
1 parent 5c71b3b commit b6c80de

5 files changed

Lines changed: 31564 additions & 0 deletions

File tree

‎apps/sim/lib/workspace-files/__integration__/file-listing-plans.integration.ts‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
/**
2+
* The plans the paged file reads must get from the live workspace-file indexes, proven on the
3+
* exact SQL the managers send: a workspace-wide page by upload date streams
4+
* `workspace_files_workspace_active_uploaded_idx`, a folder page streams
5+
* `workspace_files_workspace_folder_uploaded_idx`, a name-sorted folder page seeks the folder-name
6+
* index, a folder rollup seeks a folder-keyed index, and a name search probes
7+
* `workspace_files_workspace_name_trgm_idx`.
8+
*/
9+
import { db } from '@sim/db'
10+
import { permissions, user, workspace, workspaceFiles } from '@sim/db/schema'
11+
import { explainCaptured, type PlanNode } from '@sim/db/testing/explain'
12+
import { assertDisposableTestDatabaseUrl } from '@sim/db/testing/test-infrastructure'
13+
import { deleteWorkspaceFixture, insertWorkspaceFixture } from '@sim/db/testing/workspace-fixtures'
14+
import { generateId } from '@sim/utils/id'
15+
import { eq, sql } from 'drizzle-orm'
16+
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
17+
import {
18+
aggregateWorkspaceFileFolderUsage,
19+
queryWorkspaceFiles,
20+
} from '@/lib/uploads/contexts/workspace/file-manager'
21+
22+
const alice = generateId()
23+
const workspaceId = generateId()
24+
const folderId = generateId()
25+
26+
const scanOf = (nodes: PlanNode[], index: string) =>
27+
nodes.find((node) => node['Index Name'] === index)
28+
const sorts = (nodes: PlanNode[]) =>
29+
nodes.filter((node) => node['Node Type'] === 'Sort' || node['Node Type'] === 'Incremental Sort')
30+
31+
describe('paged file read plans in PostgreSQL', () => {
32+
beforeAll(async () => {
33+
assertDisposableTestDatabaseUrl(process.env.DATABASE_URL ?? '')
34+
await db.insert(user).values({
35+
id: alice,
36+
name: 'Plan fixture',
37+
email: `${alice}@fixture.test`,
38+
emailVerified: true,
39+
createdAt: new Date(),
40+
updatedAt: new Date(),
41+
})
42+
await insertWorkspaceFixture(db, {
43+
id: workspaceId,
44+
name: 'Plan fixture',
45+
ownerId: alice,
46+
billedAccountUserId: alice,
47+
})
48+
await db.insert(permissions).values({
49+
id: generateId(),
50+
userId: alice,
51+
entityType: 'workspace',
52+
entityId: workspaceId,
53+
permissionType: 'admin',
54+
})
55+
await db.execute(sql`
56+
INSERT INTO folder (id, name, user_id, workspace_id, resource_type)
57+
VALUES (${folderId}, 'Plans', ${alice}, ${workspaceId}, 'file')`)
58+
await db.execute(sql`
59+
INSERT INTO ${workspaceFiles} (id, key, user_id, workspace_id, folder_id, context, original_name,
60+
content_type, size_bytes, uploaded_at)
61+
SELECT 'wf_plan_' || n || '_' || ${workspaceId}, 'plan/' || n || '/' || ${workspaceId}, ${alice},
62+
${workspaceId}, CASE WHEN n % 3 = 0 THEN ${folderId} END, 'workspace',
63+
'file-' || md5(n::text) || '.txt', 'text/plain', n,
64+
timestamp '2026-01-01' + n * interval '1 second'
65+
FROM generate_series(1, 6000) AS n`)
66+
await db.execute(sql`VACUUM ANALYZE ${workspaceFiles}`)
67+
})
68+
69+
afterAll(async () => {
70+
await deleteWorkspaceFixture(db, eq(workspace.id, workspaceId))
71+
await db.delete(user).where(eq(user.id, alice))
72+
})
73+
74+
const uploadedPage = (folder: string | null | undefined, sortOrder: 'asc' | 'desc') =>
75+
explainCaptured(
76+
db,
77+
() =>
78+
queryWorkspaceFiles(workspaceId, {
79+
folderId: folder,
80+
sortBy: 'uploadedAt',
81+
sortOrder,
82+
limit: 50,
83+
after: ['2026-01-01T00:30:00.000000Z', 'wf_plan_x'],
84+
}),
85+
{ match: /order by/ }
86+
)
87+
88+
it.each(['asc', 'desc'] as const)(
89+
'streams a workspace-wide page by upload date (%s) from the active upload index, without sorting',
90+
async (sortOrder) => {
91+
const nodes = await uploadedPage(undefined, sortOrder)
92+
const scan = scanOf(nodes, 'workspace_files_workspace_active_uploaded_idx')
93+
expect(scan?.['Index Cond']).toMatch(/uploaded_at [<>]=/)
94+
expect(sorts(nodes)).toEqual([])
95+
}
96+
)
97+
98+
it.each([
99+
['the root', null, 'asc'],
100+
['the root', null, 'desc'],
101+
['a folder', folderId, 'asc'],
102+
['a folder', folderId, 'desc'],
103+
] as const)(
104+
'streams a page of %s by upload date (%s) from the folder upload index, without sorting',
105+
async (_label, folder, sortOrder) => {
106+
const nodes = await uploadedPage(folder, sortOrder)
107+
const scan = scanOf(nodes, 'workspace_files_workspace_folder_uploaded_idx')
108+
expect(scan?.['Index Cond']).toMatch(/COALESCE\(folder_id.*uploaded_at [<>]=/)
109+
expect(sorts(nodes)).toEqual([])
110+
}
111+
)
112+
113+
/**
114+
* Either folder-keyed index serves the rollup's seek — this one or the folder-name unique
115+
* index, which leads with the same `(workspace_id, coalesce(folder_id, ''))` — and the planner
116+
* picks between them on cost; what must never happen is a scan of the workspace's files.
117+
*/
118+
it('rolls folder usage up by seeking a folder-keyed index', async () => {
119+
let usage: Awaited<ReturnType<typeof aggregateWorkspaceFileFolderUsage>> | undefined
120+
const nodes = await explainCaptured(db, async () => {
121+
usage = await aggregateWorkspaceFileFolderUsage(workspaceId, [folderId])
122+
})
123+
expect(usage?.get(folderId)?.fileCount).toBe(2000)
124+
const seek = nodes.find((node) => /COALESCE\(folder_id/.test(node['Index Cond'] ?? ''))
125+
expect([
126+
'workspace_files_workspace_folder_uploaded_idx',
127+
'workspace_files_workspace_folder_name_active_unique',
128+
]).toContain(seek?.['Index Name'])
129+
})
130+
131+
/**
132+
* The folder predicate must be spelled on `coalesce(folder_id, '')`: that is the key the
133+
* `(workspace_id, coalesce(folder_id, ''), original_name)` index leads with, and a name-sorted
134+
* folder page that cannot seek it reads every live file the workspace holds.
135+
*/
136+
it.each([
137+
['the root', null],
138+
['a folder', folderId],
139+
])('seeks the folder name index for a name-sorted page of %s', async (_label, folder) => {
140+
const nodes = await explainCaptured(
141+
db,
142+
() =>
143+
queryWorkspaceFiles(workspaceId, {
144+
folderId: folder,
145+
sortBy: 'name',
146+
sortOrder: 'asc',
147+
limit: 50,
148+
after: ['file-8', 'wf_plan_x'],
149+
}),
150+
{ match: /order by/ }
151+
)
152+
const scan = scanOf(nodes, 'workspace_files_workspace_folder_name_active_unique')
153+
expect(scan?.['Index Cond']).toMatch(/COALESCE\(folder_id.*original_name >=/)
154+
})
155+
156+
it('builds the trigram name index with fastupdate off', async () => {
157+
const [index] = await db.execute<{ options: string | null }>(sql`
158+
SELECT array_to_string(reloptions, ',') AS options FROM pg_class
159+
WHERE relname = 'workspace_files_workspace_name_trgm_idx'`)
160+
expect(index?.options).toBe('fastupdate=off')
161+
})
162+
163+
it('probes the trigram name index for a substring search', async () => {
164+
const nodes = await explainCaptured(
165+
db,
166+
() =>
167+
queryWorkspaceFiles(workspaceId, {
168+
search: 'c4ca42',
169+
sortBy: 'name',
170+
sortOrder: 'asc',
171+
limit: 20,
172+
}),
173+
{ match: /order by/ }
174+
)
175+
expect(scanOf(nodes, 'workspace_files_workspace_name_trgm_idx')?.['Index Cond']).toMatch(
176+
/original_name ~~\* '%c4ca42%'/
177+
)
178+
})
179+
})

‎packages/db/migrations/0443_workspace_file_folder_upload_and_name_search_idx.sql‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
-- Two partial indexes over live workspace files, on the predicate the other live workspace-file
2+
-- indexes share, for the paged files browser:
3+
--
4+
-- workspace_files_workspace_folder_uploaded_idx serves a folder page sorted by upload date
5+
-- (equality on workspace and coalesce(folder_id, ''), then the raw (uploaded_at, id) keyset) and
6+
-- seeks a folder's live files for its rollup. size_bytes stays out of the key to keep the index
7+
-- narrow; the infrequent rollup reads it from the heap.
8+
--
9+
-- workspace_files_workspace_name_trgm_idx serves the case-insensitive substring name search
10+
-- (original_name ILIKE '%term%') within one workspace, with fastupdate off so file writes never
11+
-- stall on a foreground merge of pending entries. pg_trgm and btree_gin are created by 0313.
12+
--
13+
-- Replay-safety: workspace_files takes uploads and edits continuously, so both builds are
14+
-- CONCURRENTLY, below an embedded COMMIT. Neither index has a deployed reader yet, so a replay
15+
-- drops whatever an unjournaled earlier attempt left (valid or INVALID) and builds it again.
16+
COMMIT;--> statement-breakpoint
17+
SET lock_timeout = 0;--> statement-breakpoint
18+
-- migration-safe: new additive index with no deployed readers; drop any unjournaled prior build before rebuilding concurrently on replay.
19+
DROP INDEX CONCURRENTLY IF EXISTS "workspace_files_workspace_folder_uploaded_idx";--> statement-breakpoint
20+
CREATE INDEX CONCURRENTLY IF NOT EXISTS "workspace_files_workspace_folder_uploaded_idx" ON "workspace_files" USING btree ("workspace_id",coalesce("folder_id", ''),"uploaded_at","id") WHERE "workspace_files"."deleted_at" IS NULL AND "workspace_files"."context" = 'workspace' AND "workspace_files"."workspace_id" IS NOT NULL;--> statement-breakpoint
21+
-- migration-safe: new additive index with no deployed readers; drop any unjournaled prior build before rebuilding concurrently on replay.
22+
DROP INDEX CONCURRENTLY IF EXISTS "workspace_files_workspace_name_trgm_idx";--> statement-breakpoint
23+
CREATE INDEX CONCURRENTLY IF NOT EXISTS "workspace_files_workspace_name_trgm_idx" ON "workspace_files" USING gin ("workspace_id" text_ops,"original_name" gin_trgm_ops) WITH (fastupdate=off) WHERE "workspace_files"."deleted_at" IS NULL AND "workspace_files"."context" = 'workspace' AND "workspace_files"."workspace_id" IS NOT NULL;--> statement-breakpoint
24+
SET lock_timeout = '5s';

0 commit comments

Comments
 (0)