diff --git a/apps/web/src/page/investigations/detail/investigation.scene.test.ts b/apps/web/src/page/investigations/detail/investigation.scene.test.ts index 95240e0c63..592179d30d 100644 --- a/apps/web/src/page/investigations/detail/investigation.scene.test.ts +++ b/apps/web/src/page/investigations/detail/investigation.scene.test.ts @@ -241,6 +241,19 @@ describe("investigation detail", () => { expect(failed.model.busy).toBeNull() }) + test("a run still in flight offers no resolve, which the server would refuse", () => { + const ready = synced( + entered(), + investigationOf({ status: "investigating", report_json: null, diagnosed_at: null }), + ).model + scene( + { update: pageUpdate, view }, + given(ready), + ...canvas(), + expectView(role("button", { name: "Resolve" })).toBeDisabled(), + ) + }) + test("a failed run's primary action restarts it", () => { const ready = synced( entered(), diff --git a/apps/web/src/page/investigations/detail/view.ts b/apps/web/src/page/investigations/detail/view.ts index 890a6f8861..34acbcadbd 100644 --- a/apps/web/src/page/investigations/detail/view.ts +++ b/apps/web/src/page/investigations/detail/view.ts @@ -1,4 +1,9 @@ -import type { IssueSeverity } from "@maple/domain/http" +import { + type InvestigationStatus, + type IssueSeverity, + MANUAL_INVESTIGATION_TRANSITIONS, + RESTARTABLE_INVESTIGATION_STATUSES, +} from "@maple/domain/http" import type { V2Investigation } from "@maple/domain/http/v2" import { investigationHeadline, investigationScope } from "@maple/web-data/investigations/display" import { Mount, Submodel } from "foldkit" @@ -81,6 +86,13 @@ const titleContent = (h: H, investigation: V2Investigation): Html => { const headerActions = (h: H, model: Model, investigation: V2Investigation): Html => { const status = investigation.status const isBusy = model.busy !== null + // The server's own rules: a run in flight can be neither resolved nor restarted. + const manualMoves: ReadonlyArray = MANUAL_INVESTIGATION_TRANSITIONS[status] + const restartable: ReadonlyArray = RESTARTABLE_INVESTIGATION_STATUSES + const canResolve = manualMoves.includes("resolved") + // A run whose start never claimed it (no `started_at`) is not in flight and can be restarted. + const canRestart = + restartable.includes(status) || (status === "investigating" && investigation.started_at === null) const primary = status === "failed" ? { label: "Retry", variant: "default" as const } @@ -89,11 +101,25 @@ const headerActions = (h: H, model: Model, investigation: V2Investigation): Html : status === "resolved" ? { label: "Reopen", variant: "outline" as const } : { label: "Resolve", variant: "default" as const } + const primaryAllowed = status === "diagnosed" || status === "investigating" ? canResolve : canRestart const entries: ReadonlyArray = [ - { _tag: "Item", value: "restart", label: "Run again", icon: "ArrowPathIcon", isDisabled: isBusy }, + { + _tag: "Item", + value: "restart", + label: "Run again", + icon: "ArrowPathIcon", + isDisabled: isBusy || !canRestart, + }, ...(status === "resolved" ? [] - : [{ _tag: "Item" as const, value: "resolve", label: "Mark resolved", isDisabled: isBusy }]), + : [ + { + _tag: "Item" as const, + value: "resolve", + label: "Mark resolved", + isDisabled: isBusy || !canResolve, + }, + ]), ] return detailActions(h, [ hstack(h, { gap: "2" }, [ @@ -102,7 +128,7 @@ const headerActions = (h: H, model: Model, investigation: V2Investigation): Html { size: "sm", variant: primary.variant, - isDisabled: isBusy, + isDisabled: isBusy || !primaryAllowed, attributes: [h.OnClick(Message.ClickedPrimary())], }, [primary.label], diff --git a/docs/error-issue-lifecycle.md b/docs/error-issue-lifecycle.md index c75401079c..3437fb6d5a 100644 --- a/docs/error-issue-lifecycle.md +++ b/docs/error-issue-lifecycle.md @@ -194,6 +194,13 @@ loop in verification without a human ever seeing it. ## Rules worth not re-deriving +- **Every state write is guarded on the state it was decided from.** `applyTransition`, the + errors tick's regression and the alert/integration reopen all go through `transitionRow` + (`packages/backend/src/platform/transition-row.ts`): the UPDATE carries + `workflow_state IN ()`, so a concurrent move wins and the late writer gets a 409 + (or, for a tick, skips the row). A fixed alert or integration issue that fires again lands in + `regressed` too, never `triage`. + - **Never make the claim a step an agent has to remember.** Any new tool that starts work on an issue takes the lease itself. See above for what happened when it did not. - **An issue with a linked PR should not be closed by hand.** The merge opens the verification diff --git a/packages/backend/src/platform/transition-row.test.ts b/packages/backend/src/platform/transition-row.test.ts new file mode 100644 index 0000000000..eae4ab29fc --- /dev/null +++ b/packages/backend/src/platform/transition-row.test.ts @@ -0,0 +1,93 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import * as PG from "@maple-dev/effect-orm/postgres" +import { Effect, Schema } from "effect" +import { Investigations } from "@maple/db/tables" +import { + InvestigationFreeformSubject, + InvestigationId, + type InvestigationStatus, + OrgId, +} from "@maple/domain/http" +import { Database } from "./DatabaseLive" +import { cleanupTestDbs, createTestDb, type TestDb } from "./test-pglite" +import { transitionRow } from "./transition-row" + +const createdDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(createdDbs)) + +const ORG = Schema.decodeSync(OrgId)("org_transition_row_test") +const ID = Schema.decodeSync(InvestigationId)("00000000-0000-4000-8000-000000000001") +const NOW = Date.UTC(2026, 10, 6, 12, 0) + +const seed = Effect.gen(function* () { + const database = yield* Database + yield* database.execute((db) => + db.run( + PG.insertInto(Investigations).values({ + id: ID, + orgId: ORG, + status: "investigating", + seededBy: "system", + subjectJson: new InvestigationFreeformSubject({ + type: "freeform", + title: "seed", + prompt: "seed", + contextRefs: [], + }), + createdAt: NOW, + updatedAt: NOW, + }), + ), + ) +}) + +const move = (from: InvestigationStatus, to: InvestigationStatus) => + Effect.gen(function* () { + const database = yield* Database + return yield* database.execute((db) => + transitionRow({ table: "investigations", id: ID, from: [from], to }, (expected) => + db.run( + PG.update(Investigations) + .set({ status: to, updatedAt: NOW }) + .where(($) => [$.orgId.eq(ORG), $.id.eq(ID), $.status.in_(...expected)]) + .returning("id", "status"), + ), + ), + ) + }) + +const statusNow = Effect.gen(function* () { + const database = yield* Database + const rows = yield* database.execute((db) => + db.run( + PG.from(Investigations) + .select("status") + .where(($) => [$.orgId.eq(ORG), $.id.eq(ID)]), + ), + ) + return rows[0]?.status +}) + +describe("transitionRow", () => { + it.effect("returns the transitioned row when the state still matches", () => + Effect.gen(function* () { + yield* seed + const row = yield* move("investigating", "diagnosed") + assert.strictEqual(row.status, "diagnosed") + }).pipe(Effect.provide(createTestDb(createdDbs).layer)), + ) + + it.effect("fails the writer that read a state another writer already moved", () => + Effect.gen(function* () { + yield* seed + // Both writers read `investigating`; the first commits, the second must not overwrite it. + yield* move("investigating", "failed") + const conflict = yield* Effect.flip(move("investigating", "diagnosed")) + + assert.strictEqual(conflict._tag, "@maple/backend/TransitionConflictError") + assert.deepStrictEqual(conflict.expected, ["investigating"]) + assert.strictEqual(conflict.target, "diagnosed") + assert.strictEqual(yield* statusNow, "failed") + }).pipe(Effect.provide(createTestDb(createdDbs).layer)), + ) +}) diff --git a/packages/backend/src/platform/transition-row.ts b/packages/backend/src/platform/transition-row.ts new file mode 100644 index 0000000000..9b3b0f3c2f --- /dev/null +++ b/packages/backend/src/platform/transition-row.ts @@ -0,0 +1,47 @@ +/** + * Guarded status transitions: the UPDATE carries the state the writer read, so a concurrent + * transition wins and the late writer sees a typed conflict instead of overwriting it. + */ +import { Array as Arr, Effect, Schema } from "effect" + +export class TransitionConflictError extends Schema.TaggedError()( + "@maple/backend/TransitionConflictError", + { + message: Schema.String, + table: Schema.String, + id: Schema.String, + expected: Schema.Array(Schema.String), + target: Schema.String, + }, +) {} + +export interface TransitionRowOptions { + /** Table name, for the error and logs. */ + readonly table: string + readonly id: string + /** The states the writer read; the UPDATE only applies while the row is still in one of them. */ + readonly from: Arr.NonEmptyReadonlyArray + readonly to: S +} + +/** + * `write` must guard on ` IN (from)` plus org and id, with RETURNING. No row back fails + * with {@link TransitionConflictError}; callers remap it (a 409) or catch it as a no-op. + */ +export const transitionRow = ( + options: TransitionRowOptions, + write: (from: Arr.NonEmptyReadonlyArray) => Effect.Effect, E, R>, +): Effect.Effect => + Effect.flatMap(write(options.from), (rows) => { + const [row] = rows + if (row !== undefined) return Effect.succeed(row) + return Effect.fail( + new TransitionConflictError({ + message: `${options.table} '${options.id}' is no longer in ${Arr.join(options.from, " or ")}; it did not move to '${options.to}'.`, + table: options.table, + id: options.id, + expected: options.from, + target: options.to, + }), + ) + }) diff --git a/packages/backend/src/services/errors/ErrorIssueWorkflowService.test.ts b/packages/backend/src/services/errors/ErrorIssueWorkflowService.test.ts index e332bb0b57..8a82b48bd8 100644 --- a/packages/backend/src/services/errors/ErrorIssueWorkflowService.test.ts +++ b/packages/backend/src/services/errors/ErrorIssueWorkflowService.test.ts @@ -413,6 +413,100 @@ describe("ErrorIssueWorkflowService", () => { }).pipe(Effect.provide(makeLayer())), ) + it.effect("gives the conflict to the second of two writers that read the same state", () => + Effect.gen(function* () { + const workflow = yield* ErrorIssueWorkflowService + const database = yield* Database + const issueId = asIssueId(randomUUID()) + yield* seedIssue(issueId, { workflowState: "triage" }) + // Both callers read `triage` before either wrote. + const seenByBoth = yield* workflow.requireIssue(ORG, issueId) + + yield* workflow.applyTransition(ORG, null, seenByBoth, "cancelled") + const loser = yield* Effect.flip(workflow.applyTransition(ORG, null, seenByBoth, "done")) + assert.strictEqual(loser._tag, "@maple/http/errors/ErrorIssueTransitionError") + + const [after] = yield* database.execute((db) => + db.run( + PG.from(ErrorIssues) + .select() + .where(($) => [$.id.eq(issueId)]), + ), + ) + assert.strictEqual(after?.workflowState, "cancelled") + assert.isNull(after?.resolvedAt) + // The loser's timeline event rolled back with its UPDATE. + const events = yield* database.execute((db) => + db.run( + PG.from(ErrorIssueEvents) + .select() + .where(($) => [$.issueId.eq(issueId)]), + ), + ) + assert.deepStrictEqual( + Arr.map(events, (event) => event.toState), + ["cancelled"], + ) + }).pipe(Effect.provide(makeLayer())), + ) + + it.effect("a snooze wakeup loses to a snooze renewed after it was read", () => + Effect.gen(function* () { + const workflow = yield* ErrorIssueWorkflowService + const database = yield* Database + const now = yield* Clock.currentTimeMillis + const issueId = asIssueId(randomUUID()) + yield* seedIssue(issueId, { workflowState: "wontfix", snoozeUntil: now - 1_000 }) + const readByTick = yield* workflow.requireIssue(ORG, issueId) + // A person re-snoozes it before the tick writes. + yield* database.execute((db) => + db.run( + PG.update(ErrorIssues) + .set({ snoozeUntil: now + 3_600_000 }) + .where(($) => [$.id.eq(issueId)]), + ), + ) + + const lost = yield* Effect.flip( + workflow.applyTransition(ORG, null, readByTick, "triage", { snoozeElapsedBy: now }), + ) + assert.strictEqual(lost._tag, "@maple/http/errors/ErrorIssueTransitionError") + const after = yield* workflow.requireIssue(ORG, issueId) + assert.strictEqual(after.workflowState, "wontfix") + assert.strictEqual(after.snoozeUntil, now + 3_600_000) + }).pipe(Effect.provide(makeLayer())), + ) + + it.effect("a lease-expiry drop to todo loses to a claim taken after the lease was cleared", () => + Effect.gen(function* () { + const workflow = yield* ErrorIssueWorkflowService + const actors = yield* ErrorActorsService + const database = yield* Database + const now = yield* Clock.currentTimeMillis + const holder = yield* actors.ensureUserActor(ORG, USER) + const issueId = asIssueId(randomUUID()) + // The tick cleared the expired lease and read the issue still in progress. + yield* seedIssue(issueId, { workflowState: "in_progress" }) + const readByTick = yield* workflow.requireIssue(ORG, issueId) + // A new holder claims it before the tick writes `todo`. + yield* database.execute((db) => + db.run( + PG.update(ErrorIssues) + .set({ leaseHolderActorId: holder.id, claimedAt: now, leaseExpiresAt: now + 60_000 }) + .where(($) => [$.id.eq(issueId)]), + ), + ) + + const lost = yield* Effect.flip( + workflow.applyTransition(ORG, null, readByTick, "todo", { unleased: true }), + ) + assert.strictEqual(lost._tag, "@maple/http/errors/ErrorIssueTransitionError") + const after = yield* workflow.requireIssue(ORG, issueId) + assert.strictEqual(after.workflowState, "in_progress") + assert.strictEqual(after.leaseHolderActorId, holder.id) + }).pipe(Effect.provide(makeLayer())), + ) + // The lease used to advance only via an explicit `heartbeat` call, which agents // never made: `claim` was used 11 times in 8 days, `heartbeat` zero. Issues were // claimed, worked, and silently dropped back to `todo` when the lease lapsed. diff --git a/packages/backend/src/services/errors/ErrorIssueWorkflowService.ts b/packages/backend/src/services/errors/ErrorIssueWorkflowService.ts index dac8fb2a98..0199253012 100644 --- a/packages/backend/src/services/errors/ErrorIssueWorkflowService.ts +++ b/packages/backend/src/services/errors/ErrorIssueWorkflowService.ts @@ -42,6 +42,7 @@ import { AuditLogService } from "@maple/backend/services/audit/AuditLogService" import { CurrentAuditActor } from "@maple/backend/services/auth/audit-actor" import { SYSTEM_ERRORS_AGENT_NAME } from "@maple/backend/services/auth/system-actors" import { currentTxid, readTxid } from "@maple/backend/platform/electric-txid" +import { transitionRow } from "@maple/backend/platform/transition-row" import { ErrorActorsService } from "./ErrorActorsService" import { makeErrorDatabaseExecute } from "./error-persistence" import { issueFingerprintHashes } from "./issue-fingerprints" @@ -196,6 +197,10 @@ export interface ErrorIssueWorkflowServiceApi extends ErrorIssueWorkflowPublicAp readonly snoozeUntilMs?: number | null readonly timestamp?: number readonly payload?: StoredJsonRecord + /** Only apply while the snooze has elapsed by this time, so a renewed snooze wins. */ + readonly snoozeElapsedBy?: number + /** Only apply while nobody holds the lease, so a claim taken since the read wins. */ + readonly unleased?: boolean }, ) => Effect.Effect< ErrorIssueRow, @@ -627,13 +632,28 @@ const make: Effect.Effect< // One transaction, because a `done` that committed the issue but lost the // incident resolution or its timeline event could never be repaired: a // retry sees `fromState === toState` and returns before reaching them. + // Guarded on the state that was validated: a concurrent transition that landed first + // makes this one a 409 instead of a silent overwrite, and rolls the transaction back. yield* dbExecute((db) => db.transaction( Effect.gen(function* () { - yield* db.run( - PG.update(ErrorIssues) - .set(update) - .where(($) => [$.orgId.eq(orgId), $.id.eq(row.id)]), + yield* transitionRow( + { table: "error_issues", id: row.id, from: [fromState], to: toState }, + (from) => + db.run( + PG.update(ErrorIssues) + .set(update) + .where(($) => [ + $.orgId.eq(orgId), + $.id.eq(row.id), + $.workflowState.in_(...from), + opts.snoozeElapsedBy === undefined + ? undefined + : $.snoozeUntil.lte(opts.snoozeElapsedBy), + opts.unleased === true ? $.leaseHolderActorId.isNull() : undefined, + ]) + .returning("id"), + ), ) if (toState === "done") { yield* db.run( @@ -650,6 +670,17 @@ const make: Effect.Effect< yield* db.run(PG.insertInto(ErrorIssueEvents).values(eventInsert)) }), ), + ).pipe( + Effect.catchTag("@maple/backend/TransitionConflictError", () => + Effect.fail( + new ErrorIssueTransitionError({ + message: `The issue moved out of '${fromState}' while this change to '${toState}' was in flight. Re-read it and retry.`, + issueId: row.id, + fromState, + toState, + }), + ), + ), ) if (actorId) { yield* recordEventAudit(orgId, row.id, actorId, "state_change", { fromState, toState }) diff --git a/packages/backend/src/services/errors/ErrorsService.ts b/packages/backend/src/services/errors/ErrorsService.ts index e4d49fac47..c4abe531aa 100644 --- a/packages/backend/src/services/errors/ErrorsService.ts +++ b/packages/backend/src/services/errors/ErrorsService.ts @@ -1043,6 +1043,27 @@ const make: Effect.Effect< ) }) + /** + * Skip a system transition that lost to a concurrent move (it is only ever attempted from a + * state where the move is legal, so a refusal is the race), and keep ticking. + */ + const skipLostTransition = + (issueId: ErrorIssueId, via: string) => + ( + effect: Effect.Effect< + A, + ErrorPersistenceError | ErrorIssueNotFoundError | ErrorIssueTransitionError + >, + ) => + effect.pipe( + Effect.catchTag("@maple/http/errors/ErrorIssueTransitionError", (error) => + Effect.logInfo("Skipped a tick transition another writer beat").pipe( + Effect.annotateLogs({ issueId, via, message: error.message }), + Effect.as(undefined), + ), + ), + ) + const expireLeasesForOrg = Effect.fn("ErrorsService.expireLeases")(function* ( orgId: OrgId, nowMs: number, @@ -1054,7 +1075,8 @@ const make: Effect.Effect< yield* Effect.forEach(expired, (row) => Effect.gen(function* () { const prevActorId = row.leaseHolderActorId - yield* dbExecute((db) => + // Still expired: a heartbeat committed since the preamble read keeps the lease. + const cleared = yield* dbExecute((db) => db.run( PG.update(ErrorIssues) .set({ @@ -1063,19 +1085,25 @@ const make: Effect.Effect< claimedAt: null, updatedAt: nowMs, }) - .where(($) => [$.id.eq(row.id)]), + .where(($) => [$.orgId.eq(orgId), $.id.eq(row.id), $.leaseExpiresAt.lt(nowMs)]) + .returning("id"), ), ) + if (cleared.length === 0) return yield* recordEvent(orgId, row.id, systemActor.id, "lease_expired", { payload: { previousHolderActorId: prevActorId }, timestamp: nowMs, }) - if (row.workflowState === "in_progress") { - const refreshed = yield* requireIssue(orgId, row.id) + // Only an issue still `in_progress` drops back to `todo`; one the holder already moved + // on (to review, verifying, triage) keeps its new state, and so does one a new holder + // claimed after the clear above (guarded on the lease still being empty). + const refreshed = yield* requireIssue(orgId, row.id) + if (refreshed.workflowState === "in_progress" && refreshed.leaseHolderActorId === null) { yield* applyTransition(orgId, systemActor.id, refreshed, "todo", { payload: { viaLeaseExpiry: true }, timestamp: nowMs, - }) + unleased: true, + }).pipe(skipLostTransition(row.id, "lease_expiry")) } }), ) @@ -1190,14 +1218,18 @@ const make: Effect.Effect< systemActor, ).pipe(Effect.tapError(() => releaseTickClaim(orgId, tickWindow.claimToken, nowMs))) - const wakeCandidates = preamble.wakeCandidates - yield* Effect.forEach(wakeCandidates, (row) => + const woken = yield* Effect.forEach(preamble.wakeCandidates, (row) => applyTransition(orgId, systemActor.id, row, "triage", { payload: { viaSnoozeWakeup: true }, timestamp: nowMs, - }), + snoozeElapsedBy: nowMs, + }).pipe( + Effect.as(true), + skipLostTransition(row.id, "snooze_wakeup"), + Effect.map((applied) => applied === true), + ), ).pipe(Effect.tapError(() => releaseTickClaim(orgId, tickWindow.claimToken, nowMs))) - const issuesReopened = wakeCandidates.length + const issuesReopened = Arr.filter(woken, (applied) => applied).length const scanWindow = (endMs: number) => Effect.gen(function* () { @@ -1946,7 +1978,9 @@ const make: Effect.Effect< // One statement for every org, outside the per-org loop: an abandoned run is // rare and the matching set is almost always empty, so paying a round-trip per // org to discover that would cost more than the sweep saves. - const investigationsAbandoned = yield* dbExecute((db) => sweepAbandonedInvestigations(db, nowMs)) + const investigationsAbandoned = yield* dbExecute((db) => + sweepAbandonedInvestigations(db, nowMs), + ).pipe(Effect.map((rows) => rows.length)) if (investigationsAbandoned > 0) { yield* Effect.logWarning("Failed investigations whose agent pass never returned").pipe( Effect.annotateLogs({ count: investigationsAbandoned, budgetMs: STALE_MS }), diff --git a/packages/backend/src/services/errors/InvestigationService.test.ts b/packages/backend/src/services/errors/InvestigationService.test.ts index a122775d1a..4c9a058926 100644 --- a/packages/backend/src/services/errors/InvestigationService.test.ts +++ b/packages/backend/src/services/errors/InvestigationService.test.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto" import { afterEach, assert, describe, it } from "@effect/vitest" import { Cause, ConfigProvider, Effect, Exit, Layer, Schema, Array as Arr, Clock } from "effect" +import { TestClock } from "effect/testing" import { AiTriageEvidence, AiTriageResult, @@ -13,6 +14,8 @@ import { InvestigationSubjectSnapshot, InvestigationAgentUnavailableError, InvestigationStartFailedError, + InvestigationTransitionError, + type InvestigationProgress, OrgId, SubmitDiagnosisRequest, } from "@maple/domain/http" @@ -33,6 +36,7 @@ import { Env } from "@maple/backend/platform/Env" import { Database } from "@maple/backend/platform/DatabaseLive" import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" import { InvestigationService } from "./InvestigationService" +import { STALE_MS } from "./investigation-stale" const createdDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(createdDbs)) @@ -294,6 +298,11 @@ describe("InvestigationService", () => { const service = yield* InvestigationService const created = yield* service.createInvestigation(ORG, null, freeformRequest("status flow")) assert.strictEqual(created.status, "investigating") + yield* service.submitDiagnosis( + ORG, + created.id, + new SubmitDiagnosisRequest({ report: sampleReport() }), + ) const updated = yield* service.updateStatus(ORG, created.id, "resolved") assert.strictEqual(updated.status, "resolved") @@ -318,6 +327,139 @@ describe("InvestigationService", () => { }).pipe(Effect.provide(makeLayer())), ) + it.effect("updateStatus refuses moves a person may not make", () => + Effect.gen(function* () { + const service = yield* InvestigationService + const running = yield* service.createInvestigation(ORG, null, freeformRequest("still running")) + const refused = yield* Effect.flip(service.updateStatus(ORG, running.id, "resolved")) + assert.instanceOf(refused, InvestigationTransitionError) + assert.strictEqual((yield* service.getInvestigation(ORG, running.id)).status, "investigating") + + const diagnosed = yield* service.createInvestigation(ORG, null, freeformRequest("diagnosed")) + yield* service.submitDiagnosis( + ORG, + diagnosed.id, + new SubmitDiagnosisRequest({ report: sampleReport() }), + ) + const relabel = yield* Effect.flip(service.updateStatus(ORG, diagnosed.id, "failed")) + assert.instanceOf(relabel, InvestigationTransitionError) + + // Reopening lands back on the run's own verdict, and only there. + yield* service.updateStatus(ORG, diagnosed.id, "resolved") + const wrongReopen = yield* Effect.flip(service.updateStatus(ORG, diagnosed.id, "inconclusive")) + assert.instanceOf(wrongReopen, InvestigationTransitionError) + // The refusal names the one target this row accepts, not every verdict. + const restartByStatus = yield* Effect.flip( + service.updateStatus(ORG, diagnosed.id, "investigating"), + ) + assert.instanceOf(restartByStatus, InvestigationTransitionError) + assert.include(wrongReopen.message, "only reopen to 'diagnosed'") + assert.include(restartByStatus.message, "only reopen to 'diagnosed'") + assert.notInclude(restartByStatus.message, "inconclusive") + assert.include(restartByStatus.message, "not 'investigating'") + const reopened = yield* service.updateStatus(ORG, diagnosed.id, "diagnosed") + assert.strictEqual(reopened.status, "diagnosed") + }).pipe(Effect.provide(makeLayer())), + ) + + it.effect("reopens a rerun that failed after an earlier diagnosis as failed", () => { + const chat = chatSessionHarness() + const harness = makeHarness(chat.env, chat.chatSessions) + return Effect.gen(function* () { + const service = yield* InvestigationService + const created = yield* service.createInvestigation(ORG, null, freeformRequest("rerun fails")) + yield* service.submitDiagnosis( + ORG, + created.id, + new SubmitDiagnosisRequest({ report: sampleReport() }), + ) + yield* service.restartInvestigation(ORG, created.id) + yield* service.failInvestigation(ORG, created.id, "no_diagnosis: the pass ended in prose") + yield* service.updateStatus(ORG, created.id, "resolved") + + const stale = yield* Effect.flip(service.updateStatus(ORG, created.id, "diagnosed")) + assert.instanceOf(stale, InvestigationTransitionError) + const reopened = yield* service.updateStatus(ORG, created.id, "failed") + assert.strictEqual(reopened.status, "failed") + }).pipe(Effect.provide(harness.layer)) + }) + + it.effect("drops a diagnosis that arrives after the run failed", () => + Effect.gen(function* () { + const service = yield* InvestigationService + const created = yield* service.createInvestigation(ORG, null, freeformRequest("late report")) + yield* service.failInvestigation(ORG, created.id, "diagnosis_timeout: swept") + + const doc = yield* service.submitDiagnosis( + ORG, + created.id, + new SubmitDiagnosisRequest({ report: sampleReport() }), + ) + assert.strictEqual(doc.status, "failed") + assert.isNull(doc.report) + const partial = yield* service.submitDiagnosis( + ORG, + created.id, + new SubmitDiagnosisRequest({ report: sampleReport(), partial: true }), + ) + assert.strictEqual(partial.status, "failed") + }).pipe(Effect.provide(makeLayer())), + ) + + it.effect("refuses to restart a run that is still in flight", () => { + const chat = chatSessionHarness() + const harness = makeHarness(chat.env, chat.chatSessions) + return Effect.gen(function* () { + const service = yield* InvestigationService + const started = yield* service.createAndStartInvestigation( + ORG, + null, + freeformRequest("in flight"), + ) + const refused = yield* Effect.flip(service.restartInvestigation(ORG, started.id)) + assert.instanceOf(refused, InvestigationTransitionError) + assert.lengthOf(chat.beginTurns, 1) + }).pipe(Effect.provide(harness.layer)) + }) + + it.effect("restarts a run whose start never claimed it", () => { + const chat = chatSessionHarness() + const harness = makeHarness(chat.env, chat.chatSessions) + return Effect.gen(function* () { + const service = yield* InvestigationService + // Created but never started: `investigating` with no `startedAt`, so nothing is running. + const orphan = yield* service.createInvestigation(ORG, null, freeformRequest("orphan")) + const restarted = yield* service.restartInvestigation(ORG, orphan.id) + assert.strictEqual(restarted.status, "investigating") + assert.lengthOf(chat.beginTurns, 1) + }).pipe(Effect.provide(harness.layer)) + }) + + it.effect("the read-path sweep fails an abandoned run but spares one still recording steps", () => + Effect.gen(function* () { + const now = Date.UTC(2026, 10, 6, 12, 0) + yield* TestClock.setTime(now) + const database = yield* Database + const service = yield* InvestigationService + const alive = yield* service.createInvestigation(ORG, null, freeformRequest("long close-out")) + const dead = yield* service.createInvestigation(ORG, null, freeformRequest("abandoned")) + const age = (id: InvestigationId, progressJson: InvestigationProgress | null) => + database.execute((db) => + db.run( + PG.update(Investigations) + .set({ startedAt: now - STALE_MS - 60_000, progressJson }) + .where(($) => [$.id.eq(id)]), + ), + ) + yield* age(alive.id, { stepCount: 4, steps: [], updatedAt: now - 8_000 }) + yield* age(dead.id, null) + + // Reading the abandoned run sweeps the org; the old sweep ignored the heartbeat. + assert.strictEqual((yield* service.getInvestigation(ORG, dead.id)).status, "failed") + assert.strictEqual((yield* service.getInvestigation(ORG, alive.id)).status, "investigating") + }).pipe(Effect.provide(makeLayer())), + ) + /** * Stub `ChatSession` namespace. The autonomous turn now claims the turn on the ChatSession * Durable Object in process instead of POSTing back to chat-flue over a service binding, so the @@ -460,6 +602,7 @@ describe("InvestigationService", () => { criticalIncidentRequest("err_over_budget"), ) assert.strictEqual(started.status, "investigating") + yield* service.failInvestigation(ORG, started.id, "no_diagnosis: the pass ended in prose") const restarted = yield* service.restartInvestigation(ORG, started.id) assert.strictEqual(restarted.status, "investigating") @@ -567,18 +710,20 @@ describe("InvestigationService", () => { const failed = yield* triage(FAILED_ID) yield* service.failInvestigation(ORG, failed.id, "no_diagnosis: the pass ended in prose") const manual = yield* triage(MANUAL_ID) + yield* service.failInvestigation(ORG, manual.id, "no_diagnosis: the pass ended in prose") + // Resolving by hand afterwards does not rewrite a triage that already settled. yield* service.updateStatus(ORG, manual.id, "resolved") - assert.deepStrictEqual(yield* statuses, ["completed", "skipped", "completed"]) + assert.deepStrictEqual(yield* statuses, ["completed", "skipped", "skipped"]) // A retry is triaging again, and its diagnosis completes it. yield* service.restartInvestigation(ORG, failed.id) - assert.deepStrictEqual(yield* statuses, ["completed", "pending", "completed"]) + assert.deepStrictEqual(yield* statuses, ["completed", "pending", "skipped"]) yield* service.submitDiagnosis( ORG, failed.id, new SubmitDiagnosisRequest({ report: sampleReport() }), ) - assert.deepStrictEqual(yield* statuses, ["completed", "completed", "completed"]) + assert.deepStrictEqual(yield* statuses, ["completed", "completed", "skipped"]) }).pipe(Effect.provide(harness.layer)) }) diff --git a/packages/backend/src/services/errors/InvestigationService.ts b/packages/backend/src/services/errors/InvestigationService.ts index b927f57645..79e2ca04ad 100644 --- a/packages/backend/src/services/errors/InvestigationService.ts +++ b/packages/backend/src/services/errors/InvestigationService.ts @@ -10,6 +10,7 @@ import { InvestigationNotFoundError, InvestigationPersistenceError, InvestigationStartFailedError, + InvestigationTransitionError, InvestigationSnapshotFact, InvestigationSubjectSnapshot, InvestigationsListResponse, @@ -17,6 +18,9 @@ import { type InvestigationStatus, InvestigationSubject, type AnomalyTriageStatus, + MANUAL_INVESTIGATION_TRANSITIONS, + RESTARTABLE_INVESTIGATION_STATUSES, + reopenedInvestigationStatus, type OrgId, type SubmitDiagnosisRequest, type UserId, @@ -33,14 +37,14 @@ import { } from "@maple/backend/services/errors/apply-diagnosis" import { startInvestigationTurn } from "@maple/backend/services/errors/investigation-start" import { - STALE_MS, isInvestigationStale, - staleTimeoutMessage, storedProgressJson, + sweepAbandonedInvestigations, } from "@maple/backend/services/errors/investigation-stale" import { ChatSessions } from "@maple/backend/platform/bindings" import { Database } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" +import { transitionRow } from "@maple/backend/platform/transition-row" const decodeIdSync = Schema.decodeUnknownSync(InvestigationId) const decodeIsoSync = Schema.decodeUnknownSync(InvestigationDocument.fields.createdAt) @@ -129,6 +133,7 @@ export interface InvestigationServiceApi { InvestigationDocument, | InvestigationPersistenceError | InvestigationNotFoundError + | InvestigationTransitionError | InvestigationAgentUnavailableError | InvestigationStartFailedError | InvestigationDataCorruptionError @@ -139,7 +144,10 @@ export interface InvestigationServiceApi { status: InvestigationStatus, ) => Effect.Effect< InvestigationDocument, - InvestigationPersistenceError | InvestigationNotFoundError | InvestigationDataCorruptionError + | InvestigationPersistenceError + | InvestigationNotFoundError + | InvestigationTransitionError + | InvestigationDataCorruptionError > readonly submitDiagnosis: ( orgId: OrgId, @@ -352,9 +360,8 @@ export class InvestigationService extends Context.Service isInvestigationStale(row, nowMs) - // An anomaly's AI triage is an investigation anchored on the incident, so - // every status change of that investigation (start, restart, end, manual) - // sets the incident's triage status. Best-effort: it is display-only. + // An anomaly's AI triage is an investigation anchored on the incident: a (re)start + // re-arms it, and the run's first settled status settles it once. Display-only. const settleAnomalyTriage = ( orgId: OrgId, runs: ReadonlyArray>, @@ -370,32 +377,25 @@ export class InvestigationService extends Context.Service db.run( PG.update(AnomalyIncidents) .set({ triageStatus, updatedAt: nowMs }) - .where(($) => [$.orgId.eq(orgId), $.id.in_(first, ...rest)]), + .where(($) => [ + $.orgId.eq(orgId), + $.id.in_(first, ...rest), + triageStatus === "pending" + ? undefined + : $.triageStatus.in_("none", "pending"), + ]), ), ).pipe(Effect.ignore({ log: "Warn", message: "Failed to settle anomaly triage status" })) } const failStaleInvestigations = Effect.fnUntraced(function* (orgId: OrgId, nowMs: number) { - const failed = yield* dbExecute((db) => - db.run( - PG.update(Investigations) - .set({ - status: "failed", - error: staleTimeoutMessage(STALE_MS), - updatedAt: nowMs, - }) - .where(($) => [ - $.orgId.eq(orgId), - $.status.eq("investigating"), - $.startedAt.lt(nowMs - STALE_MS), - ]) - .returning("incidentKind", "incidentId"), - ), - ) + const failed = yield* dbExecute((db) => sweepAbandonedInvestigations(db, nowMs, orgId)) yield* settleAnomalyTriage(orgId, failed, "failed", nowMs) }) @@ -616,22 +616,53 @@ export class InvestigationService extends Context.Service - db.run( - PG.update(Investigations) - .set(($) => ({ - status: "investigating", - error: null, - startedAt: nowMs, - autonomousTurns: $.autonomousTurns.add(1), - updatedAt: nowMs, - })) - .where(($) => [$.orgId.eq(orgId), $.id.eq(id)]) - .returning("incidentKind", "incidentId"), + // A run in flight is not restarted underneath itself; a stale one was failed by the read + // above, and one whose start never claimed it (no `startedAt`) is not in flight. + const restarted = yield* dbExecute((db) => + transitionRow( + { + table: "investigations", + id, + from: RESTARTABLE_INVESTIGATION_STATUSES, + to: "investigating", + }, + (from) => + db.run( + PG.update(Investigations) + .set(($) => ({ + status: "investigating", + error: null, + startedAt: nowMs, + autonomousTurns: $.autonomousTurns.add(1), + updatedAt: nowMs, + })) + .where(($) => [ + $.orgId.eq(orgId), + $.id.eq(id), + PG.or( + $.status.in_(...from), + PG.and($.status.eq("investigating"), $.startedAt.isNull()), + ), + ]) + .returning("incidentKind", "incidentId"), + ), ), ).pipe( - Effect.tap((restarted) => settleAnomalyTriage(orgId, restarted, "investigating", nowMs)), + Effect.catchTag("@maple/backend/TransitionConflictError", () => + Effect.fail( + new InvestigationTransitionError({ + message: + existing.status === "investigating" + ? "This investigation is still running; wait for it to finish before restarting it." + : "The investigation changed while restarting it. Reload and retry.", + investigationId: id, + fromStatus: existing.status, + toStatus: "investigating", + }), + ), + ), ) + yield* settleAnomalyTriage(orgId, [restarted], "investigating", nowMs) const restarting = new InvestigationDocument({ ...existing, status: "investigating", @@ -651,20 +682,56 @@ export class InvestigationService extends Context.Service - db.run( - PG.update(Investigations) - .set({ status, updatedAt: nowMs }) - .where(($) => [$.orgId.eq(orgId), $.id.eq(id)]) - .returning("id", "incidentKind", "incidentId"), - ), - ) - if (updated.length === 0) { + const current = yield* loadRow(orgId, id) + if (!current) { return yield* new InvestigationNotFoundError({ message: `No such investigation: '${id}'`, }) } - yield* settleAnomalyTriage(orgId, updated, status, nowMs) + const fromStatus = current.status + if (fromStatus === status) return yield* documentFor(orgId, current) + const refuse = (message: string) => + new InvestigationTransitionError({ + message, + investigationId: id, + fromStatus, + toStatus: status, + }) + // A resolved run reopens only to the verdict it reached, so that is its one allowed target. + const reopened = + fromStatus === "resolved" + ? reopenedInvestigationStatus({ + failed: current.error !== null, + diagnosed: current.diagnosedAt !== null, + hasReport: current.reportJson !== null, + }) + : null + const allowed: ReadonlyArray = + reopened === null ? MANUAL_INVESTIGATION_TRANSITIONS[fromStatus] : [reopened] + if (!allowed.includes(status)) { + return yield* refuse( + fromStatus === "investigating" + ? "This investigation is still running; its status settles when the run ends." + : reopened !== null + ? `A resolved investigation can only reopen to '${reopened}', the verdict its run reached, not '${status}'.` + : `An investigation cannot move from '${fromStatus}' to '${status}' by hand. From '${fromStatus}' it can go to: ${Arr.join(allowed, ", ")}.`, + ) + } + const updated = yield* dbExecute((db) => + transitionRow({ table: "investigations", id, from: [fromStatus], to: status }, (from) => + db.run( + PG.update(Investigations) + .set({ status, updatedAt: nowMs }) + .where(($) => [$.orgId.eq(orgId), $.id.eq(id), $.status.in_(...from)]) + .returning("id", "incidentKind", "incidentId"), + ), + ), + ).pipe( + Effect.catchTag("@maple/backend/TransitionConflictError", () => + Effect.fail(refuse("The investigation changed while updating it. Reload and retry.")), + ), + ) + yield* settleAnomalyTriage(orgId, [updated], status, nowMs) const row = yield* loadRow(orgId, id) if (!row) { return yield* new InvestigationNotFoundError({ @@ -726,16 +793,18 @@ export class InvestigationService extends Context.Service - db.run( - PG.update(Investigations) - .set({ - status: "failed", - error: staleTimeoutMessage(budget), - updatedAt: nowMs, - }) - .where(($) => [ - $.orgId.eq(input.orgId), - $.id.eq(existing.id), - $.startedAt.lt(nowMs - budget), - ]), - ), - ) + yield* database.execute((db) => sweepAbandonedInvestigations(db, nowMs, input.orgId)) } return { enqueued: false, investigationId: existing.id, reason: "duplicate" as const } } diff --git a/packages/backend/src/services/errors/apply-diagnosis.ts b/packages/backend/src/services/errors/apply-diagnosis.ts index 7a4b7e2625..7675749991 100644 --- a/packages/backend/src/services/errors/apply-diagnosis.ts +++ b/packages/backend/src/services/errors/apply-diagnosis.ts @@ -13,12 +13,13 @@ * a summed fan-out). */ import * as PG from "@maple-dev/effect-orm/postgres" -import type { MapleDb } from "@maple/db/client" +import type { MapleDb, MapleDbError } from "@maple/db/client" import { ErrorIssueEvents, Investigations } from "@maple/db/tables" import { InvestigationSubjectDiscriminator, type AiTriageResult, type InvestigationConfidence, + type InvestigationStatus, type InvestigationSubjectType, type OrgId, } from "@maple/domain/http" @@ -27,6 +28,7 @@ import { createHash } from "node:crypto" import { Effect, identity, Option, Schema, Array as Arr, String as Str } from "effect" import { Database, type DatabaseError } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute } from "@maple/backend/platform/db-execute" +import { transitionRow } from "@maple/backend/platform/transition-row" import { applyTriageSeverity, type TriageActorMissingError, @@ -102,6 +104,40 @@ export interface ApplyDiagnosisInput { readonly subjectType: Option.Option } +/** Whether a diagnosis or partial actually reached the row. */ +export type DiagnosisOutcome = "applied" | "dropped" + +/** + * A diagnosis lands on a run still going, or supersedes an earlier verdict (an attended + * follow-up). A run already `failed` (the stale sweep gave up on it) or `resolved` by a person + * keeps its state: the late report is dropped, not written. + */ +const DIAGNOSABLE_STATUSES = ["investigating", "diagnosed", "inconclusive"] as const + +/** A partial only ever closes out the autonomous pass that is still running. */ +const PARTIAL_STATUSES = ["investigating"] as const + +const dropLateWrite = ( + investigationId: InvestigationId, + to: InvestigationStatus, + from: Arr.NonEmptyReadonlyArray, + write: ( + from: Arr.NonEmptyReadonlyArray, + ) => Effect.Effect, MapleDbError>, +): Effect.Effect => + transitionRow({ table: "investigations", id: investigationId, from, to }, write).pipe( + Effect.as("applied"), + Effect.catchTag("@maple/backend/TransitionConflictError", (conflict) => + Effect.gen(function* () { + yield* Effect.annotateCurrentSpan({ "maple.investigation.late_write_dropped": to }) + yield* Effect.logWarning("Dropped a late investigation write").pipe( + Effect.annotateLogs({ investigationId, to, message: conflict.message }), + ) + return "dropped" as const + }), + ), + ) + /** * Flip the row to `diagnosed` with the report attached, then — if an issue is * linked — apply the severity and record the timeline event atomically. The @@ -113,29 +149,41 @@ const writeDiagnosis = (orm: MapleDb, input: ApplyDiagnosisInput) => const confidence: InvestigationConfidence = input.report.confidence const now = input.nowMs - yield* orm.run( - PG.update(Investigations) - .set({ - status: "diagnosed", - reportJson: input.report, - // Explicit null rather than `undefined`: an unassessed report must write - // "no severity" onto the row, not silently omit the column from the UPDATE - // and leave a stale one standing. - severity: input.report.severityAssessment ?? null, - confidence, - model: input.model, - inputTokens: input.inputTokens, - outputTokens: input.outputTokens, - error: null, - diagnosedAt: now, - updatedAt: now, - }) - .where(($) => [$.orgId.eq(input.orgId), $.id.eq(input.investigationId)]), + const landed = yield* dropLateWrite( + input.investigationId, + "diagnosed", + DIAGNOSABLE_STATUSES, + (from) => + orm.run( + PG.update(Investigations) + .set({ + status: "diagnosed", + reportJson: input.report, + // Explicit null rather than `undefined`: an unassessed report must write + // "no severity" onto the row, not silently omit the column from the UPDATE + // and leave a stale one standing. + severity: input.report.severityAssessment ?? null, + confidence, + model: input.model, + inputTokens: input.inputTokens, + outputTokens: input.outputTokens, + error: null, + diagnosedAt: now, + updatedAt: now, + }) + .where(($) => [ + $.orgId.eq(input.orgId), + $.id.eq(input.investigationId), + $.status.in_(...from), + ]) + .returning("id"), + ), ) + if (landed === "dropped") return landed - if (!input.issueId) return + if (!input.issueId) return landed // See `subjectType` above: a verification's report must not re-rank the issue. - if (Option.contains(input.subjectType, "fix_verification")) return + if (Option.contains(input.subjectType, "fix_verification")) return landed const decodedIssueId = decodeIssueId(input.issueId) // Keyed by the turn: a follow-up's superseding diagnosis records its own events, a retry // does not. Without a turn, one set per investigation, as before. @@ -174,6 +222,7 @@ const writeDiagnosis = (orm: MapleDb, input: ApplyDiagnosisInput) => ) }), ) + return landed }) /** @@ -195,11 +244,11 @@ const writeDiagnosis = (orm: MapleDb, input: ApplyDiagnosisInput) => */ export const applyDiagnosisWrites: ( input: ApplyDiagnosisInput, -) => Effect.Effect = Effect.fn( +) => Effect.Effect = Effect.fn( "ApplyDiagnosis.applyDiagnosisWrites", )(function* (input) { const database = yield* Database - yield* makeDbExecute(database, "applyDiagnosisWrites", identity)((db) => writeDiagnosis(db, input)) + return yield* makeDbExecute(database, "applyDiagnosisWrites", identity)((db) => writeDiagnosis(db, input)) }) export interface ApplyInconclusiveInput { @@ -236,24 +285,31 @@ export interface ApplyInconclusiveInput { * render in a destructive box. The report replaces it. */ const writeInconclusive = (orm: MapleDb, input: ApplyInconclusiveInput) => - orm.run( - PG.update(Investigations) - .set({ - status: "inconclusive", - reportJson: input.report, - severity: null, - // Always low, whatever the report says. Nothing was established, and a - // partial that claims medium confidence in a lead is a promotion nobody - // made, reintroduced one layer down. - confidence: "low", - model: input.model, - inputTokens: input.inputTokens, - outputTokens: input.outputTokens, - error: null, - diagnosedAt: null, - updatedAt: input.nowMs, - }) - .where(($) => [$.orgId.eq(input.orgId), $.id.eq(input.investigationId)]), + dropLateWrite(input.investigationId, "inconclusive", PARTIAL_STATUSES, (from) => + orm.run( + PG.update(Investigations) + .set({ + status: "inconclusive", + reportJson: input.report, + severity: null, + // Always low, whatever the report says. Nothing was established, and a + // partial that claims medium confidence in a lead is a promotion nobody + // made, reintroduced one layer down. + confidence: "low", + model: input.model, + inputTokens: input.inputTokens, + outputTokens: input.outputTokens, + error: null, + diagnosedAt: null, + updatedAt: input.nowMs, + }) + .where(($) => [ + $.orgId.eq(input.orgId), + $.id.eq(input.investigationId), + $.status.in_(...from), + ]) + .returning("id"), + ), ) /** @@ -263,13 +319,13 @@ const writeInconclusive = (orm: MapleDb, input: ApplyInconclusiveInput) => */ export const applyInconclusiveWrites: ( input: ApplyInconclusiveInput, -) => Effect.Effect = Effect.fn("ApplyDiagnosis.applyInconclusiveWrites")( - function* (input) { - const database = yield* Database - yield* makeDbExecute( - database, - "applyInconclusiveWrites", - identity, - )((db) => writeInconclusive(db, input)) - }, -) +) => Effect.Effect = Effect.fn( + "ApplyDiagnosis.applyInconclusiveWrites", +)(function* (input) { + const database = yield* Database + return yield* makeDbExecute( + database, + "applyInconclusiveWrites", + identity, + )((db) => writeInconclusive(db, input)) +}) diff --git a/packages/backend/src/services/errors/error-tick-persistence.ts b/packages/backend/src/services/errors/error-tick-persistence.ts index 6706e56b65..40f2c06ba4 100644 --- a/packages/backend/src/services/errors/error-tick-persistence.ts +++ b/packages/backend/src/services/errors/error-tick-persistence.ts @@ -597,6 +597,65 @@ const buildNotificationRows = (input: PersistErrorTickWindowInput, payload: Noti })) } +/** + * Reopen fixed issues as `regressed`, guarded on `done`: the scan read the state with a plain + * SELECT, so a human transition committed since then (a re-triage, a `cancelled`) wins and the + * row is left alone. Returns the ids that actually reopened, plus the lost ones whose current + * state still suppresses them (closed again, or a `wontfix` under an active snooze), so the + * tick keeps them out of the incident path instead of trusting the scan's stale read. + */ +export const applyRegressions = ( + tx: MapleDb, + input: Pick, + issueIds: ReadonlyArray, +): Effect.Effect< + { readonly reopened: ReadonlySet; readonly stillSuppressed: ReadonlySet }, + MapleDbError +> => + Effect.gen(function* () { + const reopened = yield* inChunks(issueIds, (chunk) => + tx.run( + PG.update(ErrorIssues) + .set(($) => ({ + workflowState: "regressed", + resolvedAt: null, + resolvedByActorId: null, + lastRegressedAt: input.windowEndMs, + regressionCount: $.regressionCount.add(1), + // A regression on an issue retention already archived must be seen. + archivedAt: null, + updatedAt: input.windowEndMs, + })) + .where(($) => [$.orgId.eq(input.orgId), $.id.in_(...chunk), $.workflowState.eq("done")]) + .returning("id"), + ), + ) + const lost = issueIds.length - reopened.length + if (lost > 0) { + yield* Effect.annotateCurrentSpan({ "maple.error_tick.regressions_lost": lost }) + yield* Effect.logInfo("Regression lost to a concurrent transition").pipe( + Effect.annotateLogs({ orgId: input.orgId, lost }), + ) + } + const reopenedIds = new Set(Arr.map(reopened, (row) => row.id)) + const lostIds = Arr.filter(issueIds, (id) => !reopenedIds.has(id)) + const current = yield* inChunks(lostIds, (chunk) => + tx.run( + PG.from(ErrorIssues) + .select("id", "workflowState", "snoozeUntil") + .where(($) => [$.orgId.eq(input.orgId), $.id.in_(...chunk)]), + ), + ) + const stillSuppressed = Arr.filter( + current, + (row) => + row.workflowState === "done" || + (row.workflowState === "wontfix" && + (row.snoozeUntil === null || row.snoozeUntil > input.windowEndMs)), + ) + return { reopened: reopenedIds, stillSuppressed: new Set(Arr.map(stillSuppressed, (row) => row.id)) } + }) + /** * Apply one claimed, half-open evaluator window atomically. The durable cursor * advances only if every issue, incident, event, and notification-outbox write @@ -804,6 +863,19 @@ export const persistErrorTickWindow = ( ) }) + // Apply the reopen decided above. Reopening lands in `regressed`, not + // `triage`: an issue that was fixed and came back is not the same thing + // as one nobody has looked at, and flattening the two is what let the + // same bug be picked up and fixed again from scratch. + const { reopened: regressedIds, stillSuppressed } = yield* applyRegressions( + tx, + input, + Arr.map( + Arr.filter(resolvedRows, (entry) => entry.regression), + (entry) => entry.issueId, + ), + ) + for (const { row, prior, regression, suppressed, issueId } of resolvedRows) { if (!prior) { events.push( @@ -818,7 +890,9 @@ export const persistErrorTickWindow = ( ) } - const wasRegression = regression + // Only a reopen that actually applied is a regression; one that lost to a + // concurrent human transition is an ordinary occurrence on the new state. + const wasRegression = regression && regressedIds.has(issueId) if (wasRegression) { events.push( buildEvent(input, issueId, "state_change", { @@ -835,7 +909,8 @@ export const persistErrorTickWindow = ( // Counters were accumulated by the upsert above; everything downstream // of `observed` is the incident/notification/investigation path, which // a pre-fix straggler must not enter. - if (suppressed) continue + // A lost reopen is suppressed by the state it lost to, not the scan's read. + if (suppressed || stillSuppressed.has(issueId)) continue observed.push({ issueId, @@ -845,34 +920,6 @@ export const persistErrorTickWindow = ( }) } - // Apply the reopen decided above. Reopening lands in `regressed`, not - // `triage`: an issue that was fixed and came back is not the same thing - // as one nobody has looked at, and flattening the two is what let the - // same bug be picked up and fixed again from scratch. - const regressedIds = Arr.map( - Arr.filter(observed, (entry) => entry.wasRegression), - (entry) => entry.issueId, - ) - yield* Effect.forEach( - chunks(regressedIds), - (chunk) => - tx.run( - PG.update(ErrorIssues) - .set(($) => ({ - workflowState: "regressed", - resolvedAt: null, - resolvedByActorId: null, - lastRegressedAt: windowEnd, - regressionCount: $.regressionCount.add(1), - // A regression on an issue retention already archived must be seen. - archivedAt: null, - updatedAt: windowEnd, - })) - .where(($) => [$.orgId.eq(input.orgId), $.id.in_(...chunk)]), - ), - { discard: true }, - ) - // Only retention archives an issue, on the premise that it stopped occurring. // One that occurs again was retired too early: a fingerprint bump whose new // hashes are not live yet (a self-hosted warehouse that has not applied the diff --git a/packages/backend/src/services/errors/error-tick-regression-guard.test.ts b/packages/backend/src/services/errors/error-tick-regression-guard.test.ts new file mode 100644 index 0000000000..ba495e8b52 --- /dev/null +++ b/packages/backend/src/services/errors/error-tick-regression-guard.test.ts @@ -0,0 +1,93 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import * as PG from "@maple-dev/effect-orm/postgres" +import { Effect, Schema } from "effect" +import { ErrorIssues } from "@maple/db/tables" +import { ErrorIssueId, OrgId } from "@maple/domain/primitives" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import { applyRegressions } from "./error-tick-persistence" + +const createdDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(createdDbs)) + +const ORG = Schema.decodeSync(OrgId)("org_error_tick_regression_guard") +const DONE = Schema.decodeSync(ErrorIssueId)("00000000-0000-4000-8000-0000000000d1") +const CANCELLED = Schema.decodeSync(ErrorIssueId)("00000000-0000-4000-8000-0000000000c1") +const SNOOZED = Schema.decodeSync(ErrorIssueId)("00000000-0000-4000-8000-0000000000e1") +const WOKEN = Schema.decodeSync(ErrorIssueId)("00000000-0000-4000-8000-0000000000f1") +const NOW = Date.UTC(2026, 10, 6, 12, 0) + +const seed = ( + id: ErrorIssueId, + workflowState: "done" | "cancelled" | "wontfix", + snoozeUntil: number | null = null, +) => + Effect.gen(function* () { + const database = yield* Database + yield* database.execute((db) => + db.run( + PG.insertInto(ErrorIssues).values({ + id, + orgId: ORG, + fingerprintHash: `fp-${id}`, + serviceName: "checkout-api", + exceptionType: "TimeoutError", + exceptionMessage: "upstream timed out", + topFrame: "handler.ts:42", + workflowState, + snoozeUntil, + resolvedAt: workflowState === "done" ? NOW - 60_000 : null, + firstSeenAt: NOW, + lastSeenAt: NOW, + createdAt: NOW, + updatedAt: NOW, + }), + ), + ) + }) + +describe("applyRegressions", () => { + it.effect("reopens a done issue but not one a human cancelled after the scan read it", () => + Effect.gen(function* () { + yield* seed(DONE, "done") + // The scan saw both as `done`; a person cancelled this one before the tick wrote. + yield* seed(CANCELLED, "cancelled") + const database = yield* Database + + const { reopened, stillSuppressed } = yield* database.execute((db) => + applyRegressions(db, { orgId: ORG, windowEndMs: NOW }, [DONE, CANCELLED]), + ) + assert.deepStrictEqual([...reopened], [DONE]) + // A cancelled issue takes the occurrence as an ordinary one. + assert.deepStrictEqual([...stillSuppressed], []) + + const rows = yield* database.execute((db) => + db.run( + PG.from(ErrorIssues) + .select("id", "workflowState", "regressionCount", "resolvedAt") + .where(($) => [$.orgId.eq(ORG)]) + .orderBy(["id", "asc"]), + ), + ) + assert.deepStrictEqual(rows, [ + { id: CANCELLED, workflowState: "cancelled", regressionCount: 0, resolvedAt: null }, + { id: DONE, workflowState: "regressed", regressionCount: 1, resolvedAt: null }, + ]) + }).pipe(Effect.provide(createTestDb(createdDbs).layer)), + ) + + it.effect("keeps a lost reopen suppressed when the issue moved to an active wontfix snooze", () => + Effect.gen(function* () { + // The scan saw both as `done`; one was snoozed as wontfix, the other's snooze elapsed. + yield* seed(SNOOZED, "wontfix", NOW + 3_600_000) + yield* seed(WOKEN, "wontfix", NOW - 1) + const database = yield* Database + + const { reopened, stillSuppressed } = yield* database.execute((db) => + applyRegressions(db, { orgId: ORG, windowEndMs: NOW }, [SNOOZED, WOKEN]), + ) + assert.deepStrictEqual([...reopened], []) + assert.deepStrictEqual([...stillSuppressed], [SNOOZED]) + }).pipe(Effect.provide(createTestDb(createdDbs).layer)), + ) +}) diff --git a/packages/backend/src/services/errors/investigation-stale.test.ts b/packages/backend/src/services/errors/investigation-stale.test.ts index bd98fdd6f1..179e9aa5c2 100644 --- a/packages/backend/src/services/errors/investigation-stale.test.ts +++ b/packages/backend/src/services/errors/investigation-stale.test.ts @@ -156,7 +156,7 @@ describe("sweepAbandonedInvestigations", () => { const database = yield* Database const moved = yield* database.execute((db) => sweepAbandonedInvestigations(db, NOW)) - assert.strictEqual(moved, 2) + assert.strictEqual(moved.length, 2) assert.strictEqual(yield* statusOf("1"), "failed") assert.strictEqual(yield* statusOf("2"), "investigating") assert.strictEqual(yield* statusOf("3"), "failed") @@ -172,7 +172,8 @@ describe("sweepAbandonedInvestigations", () => { progress: { stepCount: 1, steps: [] }, }) const database = yield* Database - assert.strictEqual(yield* database.execute((db) => sweepAbandonedInvestigations(db, NOW)), 1) + const swept = yield* database.execute((db) => sweepAbandonedInvestigations(db, NOW)) + assert.strictEqual(swept.length, 1) assert.strictEqual(yield* statusOf("5"), "failed") }).pipe(Effect.provide(makeLayer())), ) diff --git a/packages/backend/src/services/errors/investigation-stale.ts b/packages/backend/src/services/errors/investigation-stale.ts index 898689ccea..78f4e8eaf3 100644 --- a/packages/backend/src/services/errors/investigation-stale.ts +++ b/packages/backend/src/services/errors/investigation-stale.ts @@ -6,7 +6,8 @@ */ import * as PG from "@maple-dev/effect-orm/postgres" import type { MapleDb, MapleDbError } from "@maple/db/client" -import { Investigations } from "@maple/db/tables" +import { Investigations, type InvestigationRow } from "@maple/db/tables" +import type { OrgId } from "@maple/domain/http" import { Effect, Option, Schema } from "effect" /** One agent pass. If it has not answered in this long, it is not going to. */ @@ -58,7 +59,9 @@ export const isInvestigationStale = ( } /** - * Fail every abandoned run, across every org, in one statement. + * Fail abandoned runs: every org in one statement for the tick, or one org when a read finds a + * stale row on its page. The only sweep; there used to be a second, per-org one that ignored + * the heartbeat and failed live passes in their close-out. * * Until this existed the check ran only when something happened to read the row: a detail page * load, or a second incident on the same subject re-enqueueing. An autonomous run nobody opens is @@ -66,40 +69,35 @@ export const isInvestigationStale = ( * `investigating` indefinitely and the hub filled with investigations that were never coming back. * Prod, 2026-09-19: 78 of 100 starts left no trace at all. * - * Org-agnostic and unbatched on purpose. The per-org form would be one round-trip per org per tick - * against a table where the matching set is almost always empty; this is one statement for the - * whole tick, and `status` narrows it before `started_at` does. - * * Mirrors {@link isInvestigationStale}, heartbeat included: a run still recording steps is alive - * however long it has been going, and a sweep that ran every tick rather than on a read would - * otherwise fail a pass in its close-out. `coalesce` to zero so a row whose progress somehow - * carries no `updatedAt` reads as no heartbeat and is still swept, rather than a NULL comparison - * quietly excluding it forever. + * however long it has been going. `coalesce` to zero so a row whose progress somehow carries no + * `updatedAt` reads as no heartbeat and is still swept, rather than a NULL comparison quietly + * excluding it forever. Guarded on `investigating`, so a run that settled meanwhile is untouched. * - * Returns how many rows it moved so the tick can report a number that should normally be zero. + * Returns the rows it moved: the per-org read path settles their anomaly triage, and the tick + * reports the count, which should normally be zero. */ export const sweepAbandonedInvestigations = ( orm: MapleDb, nowMs: number, -): Effect.Effect => - Effect.map( - orm.run( - PG.update(Investigations) - .set({ - status: "failed", - error: staleTimeoutMessage(STALE_MS), - updatedAt: nowMs, - }) - .where(($) => [ - $.status.eq("investigating"), - $.startedAt.lt(nowMs - STALE_MS), - PG.or( - $.progressJson.isNull(), - PG.sql - .cond`coalesce((${$.progressJson}->>'updatedAt')::bigint, 0) < ${nowMs - PROGRESS_HEARTBEAT_STALE_MS}`, - ), - ]) - .returning("id"), - ), - (rows) => rows.length, + orgId?: OrgId, +): Effect.Effect>, MapleDbError> => + orm.run( + PG.update(Investigations) + .set({ + status: "failed", + error: staleTimeoutMessage(STALE_MS), + updatedAt: nowMs, + }) + .where(($) => [ + orgId === undefined ? undefined : $.orgId.eq(orgId), + $.status.eq("investigating"), + $.startedAt.lt(nowMs - STALE_MS), + PG.or( + $.progressJson.isNull(), + PG.sql + .cond`coalesce((${$.progressJson}->>'updatedAt')::bigint, 0) < ${nowMs - PROGRESS_HEARTBEAT_STALE_MS}`, + ), + ]) + .returning("incidentKind", "incidentId"), ) diff --git a/packages/backend/src/services/errors/investigation-start.ts b/packages/backend/src/services/errors/investigation-start.ts index 8025a0665d..df412f50bf 100644 --- a/packages/backend/src/services/errors/investigation-start.ts +++ b/packages/backend/src/services/errors/investigation-start.ts @@ -70,7 +70,11 @@ export const startInvestigationTurn: ( db.run( PG.update(Investigations) .set({ status: "failed", error, updatedAt: nowMs }) - .where(($) => [$.id.eq(investigationId)]), + .where(($) => [ + $.orgId.eq(orgId), + $.id.eq(investigationId), + $.status.eq("investigating"), + ]), ), ) .pipe(Effect.asVoid) diff --git a/packages/backend/src/services/errors/issue-hub.test.ts b/packages/backend/src/services/errors/issue-hub.test.ts index 2f7b501797..f13bade6fc 100644 --- a/packages/backend/src/services/errors/issue-hub.test.ts +++ b/packages/backend/src/services/errors/issue-hub.test.ts @@ -223,7 +223,7 @@ describe("upsertAlertIssue", () => { }).pipe(Effect.provide(makeLayer())), ) - it.effect("reopens a done issue to triage with regression events", () => + it.effect("reopens a done issue as regressed with regression events", () => Effect.gen(function* () { yield* TestClock.setTime(T0) const database = yield* Database @@ -242,8 +242,11 @@ describe("upsertAlertIssue", () => { assert.strictEqual(second.action, "reopened") const issues = yield* loadIssue - assert.strictEqual(issues[0]?.workflowState, "triage") + // A fixed issue that fires again lands in `regressed`, as the errors tick does. + assert.strictEqual(issues[0]?.workflowState, "regressed") assert.isNull(issues[0]?.resolvedAt) + assert.strictEqual(issues[0]?.regressionCount, 1) + assert.strictEqual(issues[0]?.lastRegressedAt, T0 + 100_000) const events = yield* database.execute((db) => db.run( @@ -317,7 +320,7 @@ describe("upsertAlertIssue", () => { yield* database.execute((db) => db.run( PG.update(ErrorIssues) - .set({ workflowState: "wontfix", snoozeUntil: T0 + 50_000 }) + .set({ workflowState: "wontfix", snoozeUntil: T0 + 50_000, resolvedAt: T0 + 10_000 }) .where(($) => [$.id.eq(first.issueId!)]), ), ) @@ -330,6 +333,7 @@ describe("upsertAlertIssue", () => { const issues = yield* loadIssue assert.strictEqual(issues[0]?.workflowState, "triage") assert.isNull(issues[0]?.snoozeUntil) + assert.isNull(issues[0]?.resolvedAt) }).pipe(Effect.provide(makeLayer())), ) diff --git a/packages/backend/src/services/errors/issue-hub.ts b/packages/backend/src/services/errors/issue-hub.ts index 6800cb8878..1291db40ec 100644 --- a/packages/backend/src/services/errors/issue-hub.ts +++ b/packages/backend/src/services/errors/issue-hub.ts @@ -13,6 +13,7 @@ import { DateTime, Clock, Effect, Schema } from "effect" import { Database } from "@maple/backend/platform/DatabaseLive" import type { MaybeEnqueueTriageInput } from "./ai-triage-enqueue" import { issueSeverityFromAlert } from "./severity-map" +import { applyIssueReopen, issueReopenFor } from "./issue-reopen" import { SYSTEM_ALERTS_AGENT_NAME } from "@maple/backend/services/auth/system-actors" import { summarizeCause } from "@maple/backend/platform/describe-cause" @@ -284,29 +285,24 @@ export const upsertAlertIssue: ( .where(($) => [$.orgId.eq(input.orgId), $.id.eq(prior.id), $.severity.isNull()]), ), ) - const reopenFrom: WorkflowState | null = - prior.workflowState === "done" || prior.workflowState === "wontfix" - ? prior.workflowState - : null - if (reopenFrom !== null) { + const reopen = issueReopenFor(prior, input.timestamp) + const reopened = + reopen !== null && + (yield* database + .execute((db) => applyIssueReopen(db, input.orgId, prior.id, reopen, input.timestamp)) + .pipe( + Effect.as(true), + // A concurrent transition (a human re-triage, a renewed snooze) won. + Effect.catchTag("@maple/backend/TransitionConflictError", () => + Effect.succeed(false), + ), + )) + if (reopen !== null && reopened) { action = "reopened" - yield* database.execute((db) => - db.run( - PG.update(ErrorIssues) - .set({ - workflowState: "triage", - resolvedAt: null, - resolvedByActorId: null, - snoozeUntil: null, - updatedAt: input.timestamp, - }) - .where(($) => [$.orgId.eq(input.orgId), $.id.eq(prior.id)]), - ), - ) const actorId = yield* ensureSystemAlertsActor(input.orgId) yield* recordIssueEvent(input.orgId, issueId, actorId, "state_change", { - fromState: reopenFrom, - toState: "triage", + fromState: reopen.fromState, + toState: reopen.toState, payload: { viaRegression: true, incidentId: input.incidentId }, timestamp: input.timestamp, }) diff --git a/packages/backend/src/services/errors/issue-reopen.ts b/packages/backend/src/services/errors/issue-reopen.ts new file mode 100644 index 0000000000..c29dab149a --- /dev/null +++ b/packages/backend/src/services/errors/issue-reopen.ts @@ -0,0 +1,76 @@ +import * as PG from "@maple-dev/effect-orm/postgres" +import type { MapleDb, MapleDbError } from "@maple/db/client" +import { ErrorIssues, type ErrorIssueRow } from "@maple/db/tables" +import type { OrgId, WorkflowState } from "@maple/domain/http" +import { transitionRow } from "@maple/backend/platform/transition-row" + +/** + * A closed issue that fires again: a fixed (`done`) one lands in `regressed` with the same + * bookkeeping the errors tick does, and a `wontfix` whose snooze expired wakes into `triage`. + */ +export type IssueReopen = + | { readonly fromState: "done"; readonly toState: "regressed" } + | { readonly fromState: "wontfix"; readonly toState: "triage" } + +export const issueReopenFor = ( + prior: Pick, + timestamp: number, +): IssueReopen | null => { + if (prior.workflowState === "done") return { fromState: "done", toState: "regressed" } + if (prior.workflowState === "wontfix" && prior.snoozeUntil !== null && prior.snoozeUntil <= timestamp) { + return { fromState: "wontfix", toState: "triage" } + } + return null +} + +/** + * Apply a reopen, guarded on the state it was decided from. Fails with + * `TransitionConflictError` when a concurrent transition moved the issue first (or re-snoozed + * it), so the caller records no regression for a reopen that did not happen. + */ +export const applyIssueReopen = ( + db: MapleDb, + orgId: OrgId, + issueId: ErrorIssueRow["id"], + reopen: IssueReopen, + timestamp: number, +) => + transitionRow( + { table: "error_issues", id: issueId, from: [reopen.fromState], to: reopen.toState }, + (from) => + reopen.toState === "regressed" + ? db.run( + PG.update(ErrorIssues) + .set(($) => ({ + workflowState: "regressed", + resolvedAt: null, + resolvedByActorId: null, + lastRegressedAt: timestamp, + regressionCount: $.regressionCount.add(1), + // A regression on an issue retention already archived must be seen. + archivedAt: null, + updatedAt: timestamp, + })) + .where(($) => [$.orgId.eq(orgId), $.id.eq(issueId), $.workflowState.in_(...from)]) + .returning("id"), + ) + : db.run( + PG.update(ErrorIssues) + .set({ + workflowState: "triage", + snoozeUntil: null, + // Waking up reopens the issue, so it no longer reads as resolved. + resolvedAt: null, + resolvedByActorId: null, + updatedAt: timestamp, + }) + .where(($) => [ + $.orgId.eq(orgId), + $.id.eq(issueId), + $.workflowState.in_(...from), + // A snooze renewed since the read keeps the issue asleep. + $.snoozeUntil.lte(timestamp), + ]) + .returning("id"), + ), + ) diff --git a/packages/backend/src/services/integrations/planetscale/webhook-events.test.ts b/packages/backend/src/services/integrations/planetscale/webhook-events.test.ts index 94fa947915..323b9654a1 100644 --- a/packages/backend/src/services/integrations/planetscale/webhook-events.test.ts +++ b/packages/backend/src/services/integrations/planetscale/webhook-events.test.ts @@ -327,7 +327,7 @@ describe("upsertPlanetScaleIssue", () => { [first.issueId], ), ) - assert.strictEqual(reopened?.workflow_state, "triage") + assert.strictEqual(reopened?.workflow_state, "regressed") assert.strictEqual(reopened?.occurrence_count, 3) }).pipe(Effect.provide(testDb.layer)) }) @@ -414,7 +414,7 @@ describe("upsertPlanetScaleIssue", () => { [initial.issueId], ), ) - assert.strictEqual(aggregate?.workflow_state, "triage") + assert.strictEqual(aggregate?.workflow_state, "regressed") assert.strictEqual(aggregate?.occurrence_count, 3) assert.strictEqual(aggregate?.state_changes, 1) assert.strictEqual(aggregate?.regressions, 1) diff --git a/packages/backend/src/services/integrations/planetscale/webhook-events.ts b/packages/backend/src/services/integrations/planetscale/webhook-events.ts index eb214ad1aa..8fae87a1d6 100644 --- a/packages/backend/src/services/integrations/planetscale/webhook-events.ts +++ b/packages/backend/src/services/integrations/planetscale/webhook-events.ts @@ -26,6 +26,7 @@ import { Clock, Effect, Option, Result, Schema, String as Str } from "effect" import { timestampMs } from "@maple/backend/platform/time" import { Database, DatabaseError } from "@maple/backend/platform/DatabaseLive" import { secretsEqual } from "@maple/backend/services/auth/secret-equal" +import { applyIssueReopen, issueReopenFor } from "@maple/backend/services/errors/issue-reopen" /** * PlanetScale webhook event handling: signature verification, payload decode, @@ -762,27 +763,28 @@ export const upsertPlanetScaleIssue: ( .where(($) => [$.orgId.eq(input.orgId), $.id.eq(prior.id)]), ) - const reopenFrom: WorkflowState | null = - prior.workflowState === "done" || prior.workflowState === "wontfix" - ? prior.workflowState - : null - if (reopenFrom === null) return { issueId, action: "refreshed" as const } - - yield* db.run( - PG.update(ErrorIssues) - .set({ - workflowState: "triage", - resolvedAt: null, - resolvedByActorId: null, - snoozeUntil: null, - updatedAt: input.timestamp, - }) - .where(($) => [$.orgId.eq(input.orgId), $.id.eq(prior.id)]), + const reopen = issueReopenFor(prior, input.timestamp) + if (reopen === null) return { issueId, action: "refreshed" as const } + + // The row is locked FOR UPDATE above, so this guard only ever trips on a + // writer that skipped the lock; a lost reopen is then an ordinary refresh. + const reopened = yield* applyIssueReopen( + db, + input.orgId, + prior.id, + reopen, + input.timestamp, + ).pipe( + Effect.as(true), + Effect.catchTag("@maple/backend/TransitionConflictError", () => + Effect.succeed(false), + ), ) + if (!reopened) return { issueId, action: "refreshed" as const } const actorId = yield* ensureActor yield* recordEvent(issueId, actorId, "state_change", { - fromState: reopenFrom, - toState: "triage", + fromState: reopen.fromState, + toState: reopen.toState, payload: { viaRegression: true, event: input.payload.event }, }) yield* recordEvent(issueId, actorId, "regression", { diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index 4eb36e380e..a629b0f37c 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -139,6 +139,7 @@ export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ "@maple/http/errors/WarehouseValidationError", "@maple/http/investigations/InvestigationNotFoundError", "@maple/http/investigations/InvestigationQuotaError", + "@maple/http/investigations/InvestigationTransitionError", "@maple/http/investigations/InvestigationValidationError", "@maple/http/openrouter-key/OpenRouterKeyAddOnRequiredError", "@maple/http/openrouter-key/OpenRouterKeyForbiddenError", diff --git a/packages/domain/src/http/investigations.ts b/packages/domain/src/http/investigations.ts index a10d7e412e..46da17c2f8 100644 --- a/packages/domain/src/http/investigations.ts +++ b/packages/domain/src/http/investigations.ts @@ -36,6 +36,40 @@ export const InvestigationStatus = Schema.Literals([ }) export type InvestigationStatus = Schema.Schema.Type +/** + * The status moves a person may make by hand: resolve a finished run, or reopen a resolved one. + * The autonomous pass owns everything else (`investigating` and the verdict it lands on); starting + * a new pass is a restart, not a status change. + */ +export const MANUAL_INVESTIGATION_TRANSITIONS = { + investigating: [], + diagnosed: ["resolved"], + inconclusive: ["resolved"], + failed: ["resolved"], + resolved: ["diagnosed", "inconclusive", "failed"], +} as const satisfies Record> + +/** + * Where reopening a resolved investigation lands: back on the verdict its run reached, so a + * reopen cannot claim a diagnosis that never happened. `failed` is the latest run's own outcome + * (a failure records an error, a restart or a report clears it), so a rerun that failed after an + * earlier diagnosis reopens as `failed`, not on the stale report. + */ +export const reopenedInvestigationStatus = (run: { + readonly failed: boolean + readonly diagnosed: boolean + readonly hasReport: boolean +}): InvestigationStatus => + run.failed ? "failed" : run.diagnosed ? "diagnosed" : run.hasReport ? "inconclusive" : "failed" + +/** A run in flight cannot be restarted underneath itself; every settled one can. */ +export const RESTARTABLE_INVESTIGATION_STATUSES = [ + "diagnosed", + "inconclusive", + "failed", + "resolved", +] as const satisfies Arr.NonEmptyReadonlyArray + /** Who opened the investigation: a person (attended) or an incident-open trigger. */ export const InvestigationSeededBy = Schema.Literals(["user", "system"]).annotate({ identifier: "@maple/InvestigationSeededBy", @@ -427,6 +461,25 @@ export class InvestigationRejectedError extends HttpTaggedError()( + "@maple/http/investigations/InvestigationTransitionError", + { + message: Schema.String, + investigationId: InvestigationId, + fromStatus: InvestigationStatus, + toStatus: InvestigationStatus, + }, + { + status: 409, + code: "investigation_status_conflict", + title: "Investigation status cannot change", + retry: "never", + recovery: "refresh", + exposure: "public_message", + }, +) {} + /** Stored investigation data no longer decodes into its current public schema. */ export class InvestigationDataCorruptionError extends HttpTaggedError()( "@maple/http/investigations/InvestigationDataCorruptionError", diff --git a/packages/domain/src/http/v2/investigations.ts b/packages/domain/src/http/v2/investigations.ts index 75ada5dad6..a1d0730ed9 100644 --- a/packages/domain/src/http/v2/investigations.ts +++ b/packages/domain/src/http/v2/investigations.ts @@ -10,6 +10,7 @@ import { InvestigationPersistenceError, InvestigationSeededBy, InvestigationStartFailedError, + InvestigationTransitionError, InvestigationStatus, } from "../investigations" import { TraceId, UserId } from "../../primitives" @@ -427,7 +428,8 @@ export type V2InvestigationCreateParams = Schema.Schema.Type