fix(errors): settle fix verifications once, unblock the verification tick, purge issue children

docs/ci-performance.md

@@ -10,7 +10,7 @@ when services moved out of API, leaving the longest suite unsharded.
 workspace patterns and tracked manifests. It sizes Vitest shards against a 140s
 estimated test budget and packs small suites into lanes with the same budget.
 Per-file rates for the large suites are measured from CI run 35725366619
-(backend 1.6s, api 2.3s, web 0.55s, …); unknown suites get 0.6s/file, fitted to
+(api 2.3s, web 0.55s, …; backend was re-measured at 2.0s on run 42756486011); unknown suites get 0.6s/file, fitted to
 the slowest cold small-suite lanes, plus a per-suite startup cost. File counts
 include all tracked test/spec files, so a suite can be overestimated but its
 tests are never filtered. Each lane runs the original package scripts through

docs/error-issue-lifecycle.md

@@ -211,12 +211,25 @@ loop in verification without a human ever seeing it.
 
 ## Rules worth not re-deriving
 
-- **Every state write is guarded on the state it was decided from.** `applyTransition`, the
-  errors tick's regression and the alert/integration reopen all go through `transitionRow`
+- **Every state write is guarded on the state it was decided from.** `applyTransition` and the
+  alert/integration reopen go through `transitionRow`
   (`packages/backend/src/platform/transition-row.ts`): the UPDATE carries
-  `workflow_state IN (<what was read>)`, so a concurrent move wins and the late writer gets a 409
-  (or, for a tick, skips the row). A fixed alert or integration issue that fires again lands in
-  `regressed` too, never `triage`.
+  `workflow_state IN (<what was read>)`, so a concurrent move wins and the late writer gets a 409.
+  The errors tick's regression carries the same guard in its own bulk UPDATE (`applyRegressions`
+  in `error-tick-persistence.ts`, `workflow_state = 'done'`) and skips the rows it lost. A fixed
+  alert or integration issue that fires again lands in `regressed` too, never `triage`.
+- **A pull request link is verified once.** GitHub reports `merged: true` on every event after the
+  merge, so a later edit or a redelivery must not reopen a settled check. The guard is on the link
+  row, so unlinking a PR and linking it again is a new link and verifies again.
+- **One verification row cannot starve the rest.** The tick reads due rows least recently
+  touched first. A row whose issue or link is gone is settled as `abandoned`. One whose
+  occurrences cannot be read is touched and goes to the back of the queue with its window
+  unchanged: it stays `waiting` for as long as its org's warehouse keeps failing, and is tried
+  again when its turn comes round, but it no longer sits ahead of rows that can be judged.
+- **Retention deletes an issue's children by issue id.** There are no FK cascades. Incidents,
+  evaluator state, events, aliases, pull request links, verifications and escalations go with the
+  issue. Investigations stay: each keys a chat session whose storage only `apps/ai` can release.
+  Notification outbox rows in a terminal status are pruned after 30 days.
 
 - **Never make the claim a step an agent has to remember.** Any new tool that starts work on an
   issue takes the lease itself. See above for what happened when it did not.

packages/backend/src/services/errors/ErrorIssueWorkflowService.ts

@@ -59,6 +59,18 @@ const decodeIssueDateTimeSync = Schema.decodeUnknownSync(ErrorIssueDocument.fiel
 const DEFAULT_EVENTS_LIMIT = 100
 const DEFAULT_LEASE_DURATION_MS = 30 * 60_000
 
+/** The 409 for an issue somebody else holds. `row` is the holder's row, when one was read. */
+export const leaseConflict = (issueId: ErrorIssueId, row: ErrorIssueRow | null) =>
+	new ErrorIssueLeaseConflictError({
+		message: "Issue is held by another actor",
+		issueId,
+		currentHolderActorId: row?.leaseHolderActorId ?? null,
+		leaseExpiresAt:
+			row?.leaseExpiresAt == null
+				? null
+				: decodeIssueDateTimeSync(new Date(row.leaseExpiresAt).toISOString()),
+	})
+
 export interface ErrorIssueWorkflowPublicApi {
 	readonly heartbeatIssue: (
 		orgId: OrgId,
@@ -319,26 +331,33 @@ const make: Effect.Effect<
 		issueIds,
 	) => {
 		if (issueIds.length === 0) return Effect.succeed(new Set<ErrorIssueId>())
-		return Effect.all([
-			dbExecute((db) =>
-				db.run(
-					PG.from(ErrorIncidents)
-						.select("issueId")
-						.where(($) => [$.orgId.eq(orgId), $.status.eq("open"), $.issueId.in_(...issueIds)]),
+		return Effect.all(
+			[
+				dbExecute((db) =>
+					db.run(
+						PG.from(ErrorIncidents)
+							.select("issueId")
+							.where(($) => [
+								$.orgId.eq(orgId),
+								$.status.eq("open"),
+								$.issueId.in_(...issueIds),
+							]),
+					),
 				),
-			),
-			dbExecute((db) =>
-				db.run(
-					PG.from(AlertIncidents)
-						.select(($) => ({ issueId: $.errorIssueId }))
-						.where(($) => [
-							$.orgId.eq(orgId),
-							$.status.eq("open"),
-							$.errorIssueId.in_(...issueIds),
-						]),
+				dbExecute((db) =>
+					db.run(
+						PG.from(AlertIncidents)
+							.select(($) => ({ issueId: $.errorIssueId }))
+							.where(($) => [
+								$.orgId.eq(orgId),
+								$.status.eq("open"),
+								$.errorIssueId.in_(...issueIds),
+							]),
+					),
 				),
-			),
-		]).pipe(
+			],
+			{ concurrency: "unbounded" },
+		).pipe(
 			Effect.map(
 				([errorRows, alertRows]) =>
 					new Set([
@@ -364,28 +383,31 @@ const make: Effect.Effect<
 			if (issueIds.length === 0) return rollups
 			const upsert = (issueId: ErrorIssueId, patch: Partial<IssueActivityRollup>) =>
 				rollups.set(issueId, { ...(rollups.get(issueId) ?? EMPTY_ACTIVITY), ...patch })
-			const [commentRows, prRows] = yield* Effect.all([
-				dbExecute((db) =>
-					db.run(
-						PG.from(ErrorIssueEvents)
-							.select(($) => ({ issueId: $.issueId, count: PG.count() }))
-							.where(($) => [
-								$.orgId.eq(orgId),
-								$.issueId.in_(...issueIds),
-								$.type.in_("comment", "agent_note"),
-							])
-							.groupBy("issueId"),
+			const [commentRows, prRows] = yield* Effect.all(
+				[
+					dbExecute((db) =>
+						db.run(
+							PG.from(ErrorIssueEvents)
+								.select(($) => ({ issueId: $.issueId, count: PG.count() }))
+								.where(($) => [
+									$.orgId.eq(orgId),
+									$.issueId.in_(...issueIds),
+									$.type.in_("comment", "agent_note"),
+								])
+								.groupBy("issueId"),
+						),
 					),
-				),
-				dbExecute((db) =>
-					db.run(
-						PG.from(ErrorIssuePullRequests)
-							.select(($) => ({ issueId: $.issueId, state: $.state, count: PG.count() }))
-							.where(($) => [$.orgId.eq(orgId), $.issueId.in_(...issueIds)])
-							.groupBy("issueId", "state"),
+					dbExecute((db) =>
+						db.run(
+							PG.from(ErrorIssuePullRequests)
+								.select(($) => ({ issueId: $.issueId, state: $.state, count: PG.count() }))
+								.where(($) => [$.orgId.eq(orgId), $.issueId.in_(...issueIds)])
+								.groupBy("issueId", "state"),
+						),
 					),
-				),
-			])
+				],
+				{ concurrency: "unbounded" },
+			)
 			for (const row of commentRows) upsert(row.issueId, { commentCount: row.count })
 			for (const row of prRows) {
 				if (row.state === "open") upsert(row.issueId, { openPullRequestCount: row.count })
@@ -397,12 +419,22 @@ const make: Effect.Effect<
 	const hydrateNonEmpty = (orgId: OrgId, rows: Arr.NonEmptyReadonlyArray<ErrorIssueRow>) =>
 		Effect.gen(function* () {
 			const issueIds = Arr.map(rows, (row) => row.id)
-			const openSet = yield* issuesWithOpenIncidents(orgId, issueIds)
-			const activityMap = yield* issueActivityRollups(orgId, issueIds)
-			const hashesByIssue = yield* dbExecute((db) => issueFingerprintHashes(db, orgId, rows))
-			const actorMap = yield* actorsService.collectActorDocs(
-				orgId,
-				Arr.flatMap(rows, (row) => [row.assignedActorId ?? null, row.leaseHolderActorId ?? null]),
+			// Independent reads, run on every list call and after every mutation.
+			// The invocation's pool bounds how many are really in flight.
+			const [openSet, activityMap, hashesByIssue, actorMap] = yield* Effect.all(
+				[
+					issuesWithOpenIncidents(orgId, issueIds),
+					issueActivityRollups(orgId, issueIds),
+					dbExecute((db) => issueFingerprintHashes(db, orgId, rows)),
+					actorsService.collectActorDocs(
+						orgId,
+						Arr.flatMap(rows, (row) => [
+							row.assignedActorId ?? null,
+							row.leaseHolderActorId ?? null,
+						]),
+					),
+				],
+				{ concurrency: "unbounded" },
 			)
 			return Arr.map(rows, (row) =>
 				rowToIssue(
@@ -689,14 +721,6 @@ const make: Effect.Effect<
 		return yield* requireIssue(orgId, row.id)
 	})
 
-	const leaseConflict = (issueId: ErrorIssueId, row: ErrorIssueRow | null) =>
-		new ErrorIssueLeaseConflictError({
-			message: "Issue is held by another actor",
-			issueId,
-			currentHolderActorId: row?.leaseHolderActorId ?? null,
-			leaseExpiresAt: row?.leaseExpiresAt == null ? null : isoFromMs(row.leaseExpiresAt),
-		})
-
 	const heartbeatIssue: ErrorIssueWorkflowServiceApi["heartbeatIssue"] = Effect.fn(
 		"ErrorIssueWorkflowService.heartbeatIssue",
 	)(function* (orgId, actorId, issueId) {