Skip to content

Commit 3ff8a4a

Browse files
committed
fix(db): retry public share backfill contention without losing progress
1 parent 3463c0b commit 3ff8a4a

2 files changed

Lines changed: 238 additions & 30 deletions

File tree

‎packages/db/public-share-ownership.integration.ts‎

Lines changed: 203 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
import { readFileSync } from 'node:fs'
22
import { mkdir, writeFile } from 'node:fs/promises'
33
import { dirname } from 'node:path'
4+
import {
5+
backfillPublicShareEntities,
6+
backfillPublicShareEntitiesMigration,
7+
} from '@sim/db/script-migrations/0034_backfill_public_share_entities'
8+
import { runScriptMigrations } from '@sim/db/script-migrations/index'
49
import { readTestDatabaseUrl } from '@sim/db/testing/test-infrastructure'
510
import { createDeferred } from '@sim/testing/helpers/deferred'
611
import { getErrorMessage } from '@sim/utils/errors'
@@ -19,22 +24,26 @@ const migrations = [
1924
].map((name) => readFileSync(new URL(`./migrations/${name}`, import.meta.url), 'utf8'))
2025
const checks: { name: string; status: 'passed' | 'failed'; durationMs: number; error?: string }[] =
2126
[]
22-
function check(name: string, run: () => Promise<void>) {
23-
it(name, async () => {
24-
const started = performance.now()
25-
try {
26-
await run()
27-
checks.push({ name, status: 'passed', durationMs: performance.now() - started })
28-
} catch (error) {
29-
checks.push({
30-
name,
31-
status: 'failed',
32-
durationMs: performance.now() - started,
33-
error: getErrorMessage(error),
34-
})
35-
throw error
36-
}
37-
}, 10_000)
27+
function check(name: string, run: () => Promise<void>, timeout = 10_000) {
28+
it(
29+
name,
30+
async () => {
31+
const started = performance.now()
32+
try {
33+
await run()
34+
checks.push({ name, status: 'passed', durationMs: performance.now() - started })
35+
} catch (error) {
36+
checks.push({
37+
name,
38+
status: 'failed',
39+
durationMs: performance.now() - started,
40+
error: getErrorMessage(error),
41+
})
42+
throw error
43+
}
44+
},
45+
timeout
46+
)
3847
}
3948

4049
describe('Public share canonical ownership in PostgreSQL', () => {
@@ -108,6 +117,8 @@ describe('Public share canonical ownership in PostgreSQL', () => {
108117
workspace_id text NOT NULL REFERENCES workspace(id) ON DELETE CASCADE,
109118
created_by text REFERENCES "user"(id) ON DELETE SET NULL,
110119
token text NOT NULL UNIQUE, is_active boolean NOT NULL DEFAULT true,
120+
auth_type text NOT NULL DEFAULT 'password', password text DEFAULT 'fixture-hash',
121+
allowed_emails json DEFAULT '["reader@example.test"]',
111122
updated_at timestamp NOT NULL DEFAULT '2026-10-03 01:02:03.456',
112123
UNIQUE(resource_type, resource_id)
113124
)`
@@ -125,10 +136,11 @@ describe('Public share canonical ownership in PostgreSQL', () => {
125136
}
126137
await sql`CREATE TABLE workspace_file_secret_provenance (file_id text PRIMARY KEY, content_updated_at timestamp NOT NULL)`
127138
for (const migration of migrations) await applyMigration(migration)
139+
await sql`CREATE TABLE script_migrations (name text PRIMARY KEY, applied_at timestamptz DEFAULT now())`
128140
})
129141

130142
beforeEach(async () => {
131-
await sql`TRUNCATE workspace_file_secret_provenance, public_share, workspace_file_version, workspace_files, folder, project, workspace, organization, "user"`
143+
await sql`TRUNCATE script_migrations, workspace_file_secret_provenance, public_share, workspace_file_version, workspace_files, folder, project, workspace, organization, "user"`
132144
await sql`INSERT INTO "user" VALUES ('user-a'), ('user-b')`
133145
await sql`INSERT INTO workspace VALUES ('workspace-a'), ('workspace-b'), ('same-id')`
134146
await sql`INSERT INTO organization VALUES ('organization-a')`
@@ -304,9 +316,6 @@ describe('Public share canonical ownership in PostgreSQL', () => {
304316
}
305317
const before =
306318
await sql`SELECT id, resource_type, resource_id, workspace_id, token, is_active, updated_at FROM public_share ORDER BY id`
307-
const { backfillPublicShareEntities } = await import(
308-
'./script-migrations/0034_backfill_public_share_entities'
309-
)
310319
const first = await backfillPublicShareEntities(sql)
311320
expect(first).toMatchObject({ backfilled: 502, unresolved: 3 })
312321
expect(first.sampleShareIds.sort()).toEqual(['foreign', 'missing', 'unknown'])
@@ -322,6 +331,180 @@ describe('Public share canonical ownership in PostgreSQL', () => {
322331
}
323332
)
324333

334+
async function seedLegacyFileShares() {
335+
await sql`INSERT INTO workspace_files (id, context, user_id, workspace_id, original_name, deleted_at)
336+
SELECT 'file-' || lpad(n::text, 4, '0'), 'workspace', 'user-a', 'workspace-a',
337+
'file-' || n, CASE WHEN n % 10 = 0 THEN NULL ELSE timestamp '2026-10-01 01:02:03.456' END
338+
FROM generate_series(1, 1001) n`
339+
await sql`ALTER TABLE public_share DISABLE TRIGGER public_share_entity_binding`
340+
try {
341+
await sql`INSERT INTO public_share (id, resource_type, resource_id, workspace_id, token, is_active)
342+
SELECT 'share-' || lpad(n::text, 4, '0'), 'file', 'file-' || lpad(n::text, 4, '0'),
343+
'workspace-a', 'token-' || n, n % 2 = 0 FROM generate_series(1, 1001) n`
344+
} finally {
345+
await sql`ALTER TABLE public_share ENABLE TRIGGER public_share_entity_binding`
346+
}
347+
}
348+
349+
async function retainedFileShareState() {
350+
return sql`SELECT
351+
(SELECT md5(string_agg((to_jsonb(s) - 'entity_type' - 'entity_id')::text, '' ORDER BY id))
352+
FROM public_share s WHERE id LIKE 'share-%') AS shares,
353+
(SELECT md5(string_agg(to_jsonb(f)::text, '' ORDER BY id))
354+
FROM workspace_files f WHERE id LIKE 'file-%') AS files`
355+
}
356+
357+
async function committedShareCount() {
358+
const [row] = await sql<{ count: number }[]>`SELECT count(*)::integer AS count
359+
FROM public_share WHERE entity_type = 'workspace' AND entity_id = workspace_id`
360+
return row.count
361+
}
362+
363+
for (const target of ['file', 'share'] as const) {
364+
for (const persistent of [false, true]) {
365+
check(
366+
`backfill ${persistent ? 'stops after bounded retries under persistent' : 'recovers after temporary'} ${target} lock contention without losing committed batches`,
367+
async () => {
368+
await seedLegacyFileShares()
369+
const before = await retainedFileShareState()
370+
const firstRollback = createDeferred<void>()
371+
let rollbacks = 0
372+
const worker = postgres(databaseUrl, {
373+
max: 1,
374+
onnotice: () => undefined,
375+
connection: { search_path: schemaName },
376+
debug: (_connection, query) => {
377+
if (query.toLowerCase() === 'rollback') {
378+
rollbacks += 1
379+
firstRollback.resolve()
380+
}
381+
},
382+
})
383+
const held = createDeferred<void>()
384+
const release = createDeferred<void>()
385+
const holding = sql.begin(async (tx) => {
386+
if (target === 'file')
387+
await tx`SELECT id FROM workspace_files WHERE id = 'file-0501' FOR UPDATE`
388+
else await tx`SELECT id FROM public_share WHERE id = 'share-0501' FOR UPDATE`
389+
held.resolve()
390+
await release.promise
391+
})
392+
await Promise.race([held.promise, holding])
393+
const running = runScriptMigrations(worker, [backfillPublicShareEntitiesMigration]).then(
394+
() => null,
395+
(error: unknown) => error
396+
)
397+
try {
398+
if (!persistent) {
399+
await Promise.race([firstRollback.promise, running])
400+
expect(await committedShareCount()).toBe(500)
401+
expect(await sql`SELECT name FROM script_migrations`).toEqual([])
402+
release.resolve()
403+
await holding
404+
}
405+
const error = await running
406+
if (persistent) {
407+
expect(error).toMatchObject({ code: '55P03' })
408+
expect(rollbacks).toBe(3)
409+
expect(await committedShareCount()).toBe(500)
410+
expect(await sql`SELECT name FROM script_migrations`).toEqual([])
411+
release.resolve()
412+
await holding
413+
await runScriptMigrations(worker, [backfillPublicShareEntitiesMigration])
414+
} else {
415+
expect(error).toBeNull()
416+
expect(rollbacks).toBe(1)
417+
}
418+
expect(await committedShareCount()).toBe(1001)
419+
expect(await retainedFileShareState()).toEqual(before)
420+
const receipts = await sql`SELECT * FROM script_migrations`
421+
expect(receipts).toHaveLength(1)
422+
expect(receipts[0].name).toBe(backfillPublicShareEntitiesMigration.name)
423+
await runScriptMigrations(worker, [backfillPublicShareEntitiesMigration])
424+
expect(await sql`SELECT * FROM script_migrations`).toEqual(receipts)
425+
} finally {
426+
release.resolve()
427+
await holding
428+
await running
429+
await worker.end({ timeout: 1 })
430+
}
431+
},
432+
40_000
433+
)
434+
}
435+
}
436+
437+
check(
438+
'cancelled backfill rolls back its incomplete batch, preserves progress, and resumes without missing concurrent legacy inserts',
439+
async () => {
440+
await seedLegacyFileShares()
441+
const before = await retainedFileShareState()
442+
await sql`CREATE SEQUENCE backfill_write_attempts`
443+
await sql`CREATE FUNCTION pause_backfill_batch() RETURNS trigger LANGUAGE plpgsql AS $$
444+
BEGIN
445+
PERFORM nextval('backfill_write_attempts');
446+
IF NEW.id = 'share-0750' THEN PERFORM pg_advisory_xact_lock(86090034); END IF;
447+
RETURN NEW;
448+
END
449+
$$`
450+
await sql`CREATE TRIGGER pause_backfill_batch BEFORE UPDATE ON public_share
451+
FOR EACH ROW EXECUTE FUNCTION pause_backfill_batch()`
452+
const worker = postgres(databaseUrl, {
453+
max: 1,
454+
onnotice: () => undefined,
455+
connection: { search_path: schemaName },
456+
})
457+
const [connection] = await worker<{ pid: number }[]>`SELECT pg_backend_pid() AS pid`
458+
const held = createDeferred<number>()
459+
const release = createDeferred<void>()
460+
const holding = sql.begin(async (tx) => {
461+
await tx`SELECT pg_advisory_xact_lock(86090034)`
462+
const [holder] = await tx<{ pid: number }[]>`SELECT pg_backend_pid() AS pid`
463+
held.resolve(holder.pid)
464+
await release.promise
465+
})
466+
const holderPid = await Promise.race([held.promise, holding.then(() => 0)])
467+
const running = runScriptMigrations(worker, [backfillPublicShareEntitiesMigration]).then(
468+
() => null,
469+
(error: unknown) => error
470+
)
471+
try {
472+
expect(await waitForDatabaseLock(connection.pid, holderPid)).toBe(true)
473+
expect(await committedShareCount()).toBe(500)
474+
const [attempts] = await sql<
475+
{ count: number }[]
476+
>`SELECT last_value::integer AS count FROM backfill_write_attempts`
477+
expect(attempts.count).toBeGreaterThan(501)
478+
await sql`INSERT INTO public_share (id, resource_type, resource_id, workspace_id, token)
479+
VALUES ('before-cursor', 'folder', 'concurrent-folder', 'workspace-a', 'concurrent-token')`
480+
expect(
481+
await sql`SELECT entity_type, entity_id FROM public_share WHERE id = 'before-cursor'`
482+
).toEqual([{ entity_type: 'workspace', entity_id: 'workspace-a' }])
483+
await sql`SELECT pg_cancel_backend(${connection.pid})`
484+
expect(await running).toMatchObject({ code: '57014' })
485+
expect(await committedShareCount()).toBe(501)
486+
expect(await sql`SELECT name FROM script_migrations`).toEqual([])
487+
expect(await retainedFileShareState()).toEqual(before)
488+
release.resolve()
489+
await holding
490+
await runScriptMigrations(worker, [backfillPublicShareEntitiesMigration])
491+
expect(await committedShareCount()).toBe(1002)
492+
expect(await retainedFileShareState()).toEqual(before)
493+
expect(await sql`SELECT name FROM script_migrations`).toEqual([
494+
{ name: backfillPublicShareEntitiesMigration.name },
495+
])
496+
} finally {
497+
release.resolve()
498+
await holding
499+
await running
500+
await worker.end({ timeout: 1 })
501+
await sql`DROP TRIGGER pause_backfill_batch ON public_share`
502+
await sql`DROP FUNCTION pause_backfill_batch()`
503+
await sql`DROP SEQUENCE backfill_write_attempts`
504+
}
505+
}
506+
)
507+
325508
for (const isolation of ['read committed', 'repeatable read'] as const) {
326509
check(
327510
`${isolation}: file deletion waits for share insertion and cannot leave an orphan`,

‎packages/db/script-migrations/0034_backfill_public_share_entities.ts‎

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
import type { ScriptMigration } from '@sim/db/script-migrations/types'
22
import { createLogger } from '@sim/logger'
3+
import { getTransientDatabaseFailure } from '@sim/utils/errors'
4+
import { sleep } from '@sim/utils/helpers'
5+
import { backoffWithJitter } from '@sim/utils/retry'
36
import type { Sql } from 'postgres'
47

58
const logger = createLogger('PublicShareEntityBackfill')
69
const BATCH_SIZE = 500
10+
const MAX_BATCH_ATTEMPTS = 3
711
const MAX_REPAIR_SAMPLES = 10
812

913
/** Stamps verified legacy share owners without repairing ambiguous targets or changing tokens. */
@@ -18,21 +22,23 @@ export async function backfillPublicShareEntities(sql: Sql) {
1822
ORDER BY id LIMIT ${BATCH_SIZE}
1923
`
2024
if (!rows.length) break
21-
afterId = rows[rows.length - 1].id
2225
const ids = rows.map((row) => row.id)
2326
const fileIds = [
2427
...new Set(rows.filter((row) => row.resource_type === 'file').map((row) => row.resource_id)),
2528
].sort()
26-
const updated = await sql.begin(async (tx) => {
27-
await tx`SET LOCAL lock_timeout = '5s'`
28-
await tx`SET LOCAL statement_timeout = '30s'`
29-
// Target locks precede share locks, as they do for insertion and target retirement.
30-
if (fileIds.length)
31-
await tx`
29+
const started = performance.now()
30+
for (let attempt = 1; ; attempt++) {
31+
try {
32+
const updated = await sql.begin(async (tx) => {
33+
await tx`SELECT set_config('lock_timeout', '5s', true),
34+
set_config('statement_timeout', '30s', true)`
35+
// Target locks precede share locks, as they do for insertion and target retirement.
36+
if (fileIds.length)
37+
await tx`
3238
SELECT id FROM workspace_files WHERE id = ANY(${fileIds}::text[])
3339
ORDER BY id FOR UPDATE
3440
`
35-
return tx<{ id: string }[]>`
41+
return tx<{ id: string }[]>`
3642
UPDATE public_share share
3743
SET entity_type = 'workspace', entity_id = share.workspace_id
3844
WHERE share.id = ANY(${ids}::text[]) AND share.entity_type IS NULL AND share.entity_id IS NULL
@@ -48,8 +54,27 @@ export async function backfillPublicShareEntities(sql: Sql) {
4854
))
4955
RETURNING share.id
5056
`
51-
})
52-
backfilled += updated.length
57+
})
58+
backfilled += updated.length
59+
afterId = rows[rows.length - 1].id
60+
logger.info('Public share owner batch committed', {
61+
candidates: rows.length,
62+
backfilled: updated.length,
63+
attempts: attempt,
64+
durationMs: Math.round(performance.now() - started),
65+
})
66+
break
67+
} catch (error) {
68+
const failure = getTransientDatabaseFailure(error)
69+
if (attempt >= MAX_BATCH_ATTEMPTS || (failure !== 'capacity' && failure !== 'conflict'))
70+
throw error
71+
logger.warn('Retrying public share owner batch after transient contention', {
72+
attempt,
73+
failure,
74+
})
75+
await sleep(backoffWithJitter(attempt, null, { baseMs: 250, maxMs: 1000 }))
76+
}
77+
}
5378
}
5479

5580
let unresolved = 0

0 commit comments

Comments
 (0)