diff --git a/packages/cli/src/__tests__/activation-command.test.ts b/packages/cli/src/__tests__/activation-command.test.ts index 6a6978daa8..e3b125cab1 100644 --- a/packages/cli/src/__tests__/activation-command.test.ts +++ b/packages/cli/src/__tests__/activation-command.test.ts @@ -461,6 +461,59 @@ describe('maka activate JSONL protocol', () => { }); }); + test('blocks a completed invocation whose stream carried a boundary failure', async () => { + // Guards the current contract: a completed invocation whose stream carried + // a boundary failure reports `blocked` / `permission_required` with exit 3. + // The `maka activate` transition itself (main completed with exit 0 when + // the classifier cleared `recovered`) is not regression-coverable after + // the deletion: `recovered` no longer exists in the outcome type, so an + // injected `MakaRunOutcome` cannot express the old shape. + const lines: string[] = []; + const boundaryFailure = { + kind: 'text', + text: 'Write requires an approved session sandbox boundary expansion.', + sandboxFailure: { + reason: 'sandbox_boundary_required', + requiredExpansion: { + filesystem: { + entries: [{ path: '/tmp/output', access: 'write', scope: 'subtree' }], + }, + }, + }, + } as const; + const result = await runMakaActivationCli( + [ + '--state-root', + ROOTS.stateRoot, + '--workspace-root', + ROOTS.workspaceRoot, + '--config-root', + ROOTS.configRoot, + ], + { + ...fakeDeps({ + result: completedResult(), + events: [ + { + type: 'tool_result', + id: 'event-boundary-result', + turnId: 'turn-1', + ts: 1, + toolUseId: 'tool-boundary', + isError: true, + content: boundaryFailure, + }, + ], + }), + writeStdout: (text) => lines.push(text.trim()), + }, + ); + + assert.equal(result, 3); + assert.equal(JSON.parse(lines.at(-1)!).status, 'blocked'); + assert.equal(JSON.parse(lines.at(-1)!).reason, 'permission_required'); + }); + test('retries non-permission blocked sessions instead of requesting permission', async () => { for (const blockedReason of ['auth', 'tool_failed'] as const) { const lines: string[] = []; diff --git a/packages/cli/src/__tests__/runtime-host-run-command.test.ts b/packages/cli/src/__tests__/runtime-host-run-command.test.ts index c0173f2f6e..fc13e5b2eb 100644 --- a/packages/cli/src/__tests__/runtime-host-run-command.test.ts +++ b/packages/cli/src/__tests__/runtime-host-run-command.test.ts @@ -313,6 +313,50 @@ describe('Runtime Host maka run adapter', () => { ); }); + test('keeps the boundary unresolved when a later same-named call succeeds (live)', async () => { + const fixture = runFixture({ + turnEvents: sandboxBoundaryEvents('turn-1', 'step-1', 'step-2', 'Partial answer', 'Read'), + }); + + const exitCode = await runFixtureCommand(fixture, ['accept same-name different target']); + + assert.equal(exitCode, 1); + }); + + test('keeps the boundary unresolved when a later same-named call succeeds (durable)', async () => { + const fixture = runFixture({ + graph: true, + finalMessages: sandboxBoundaryMessages('step-1', 'step-2', 'Read'), + }); + + const exitCode = await runFixtureCommand(fixture, [ + 'accept durable same-name different target', + '--graph', + ]); + + assert.equal(exitCode, 1); + }); + + test('returns exit code 1 when a denied widening precedes any later tool success', async () => { + const stderr: string[] = []; + const fixture = runFixture({ + turnEvents: deniedWideningEvents('turn-1'), + }); + + const exitCode = await runFixtureCommand( + fixture, + ['accept post-denial success'], + () => {}, + (text) => stderr.push(text), + ); + + assert.equal(exitCode, 1); + assert.equal( + stderr.join(''), + 'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n', + ); + }); + test('returns exit code 1 when reconnect restores a missed sandbox failure', async () => { let publishReplacement = () => {}; const fixture = runFixture({ @@ -351,18 +395,26 @@ describe('Runtime Host maka run adapter', () => { assert.equal(stdout.join(''), 'Final graph answer\n'); }); - test('returns exit code 0 when a root Graph boundary failure recovers', async () => { + test('keeps a root Graph boundary failure unresolved even when a later same-named call succeeds', async () => { const stdout: string[] = []; + const stderr: string[] = []; const fixture = runFixture({ graph: true, turnEvents: sandboxBoundaryEvents('turn-1', 'step-1', 'step-2', 'Recovered answer'), }); - const exitCode = await runFixtureCommand(fixture, ['recover once', '--graph'], (text) => - stdout.push(text), + const exitCode = await runFixtureCommand( + fixture, + ['recover once', '--graph'], + (text) => stdout.push(text), + (text) => stderr.push(text), ); - assert.equal(exitCode, 0); - assert.equal(stdout.join(''), 'Final graph answer\n'); + assert.equal(exitCode, 1); + assert.equal(stdout.join(''), ''); + assert.equal( + stderr.join(''), + 'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n', + ); }); test('returns exit code 1 when a same-step Graph sibling succeeds after a sandbox failure', async () => { @@ -465,7 +517,7 @@ describe('Runtime Host maka run adapter', () => { assert.equal(observed.at(-1)?.finalOutput, 'Final graph answer'); }); - test('reports a recovered sandbox boundary from live and durable Turns', async () => { + test('keeps the sandbox boundary unresolved across live and durable Turns', async () => { const live = await observeFixtureOutcome({ turnEvents: sandboxBoundaryEvents('turn-1', 'step-1', 'step-2', 'Recovered answer'), }); @@ -474,8 +526,8 @@ describe('Runtime Host maka run adapter', () => { finalMessages: sandboxBoundaryMessages('step-1', 'step-2'), }); - assert.equal(live.sandboxBoundary, 'recovered'); - assert.equal(durable.sandboxBoundary, 'recovered'); + assert.equal(live.sandboxBoundary, 'unresolved'); + assert.equal(durable.sandboxBoundary, 'unresolved'); }); test('leaves sandbox failures unresolved when their provider steps are unavailable', async () => { @@ -491,7 +543,7 @@ describe('Runtime Host maka run adapter', () => { assert.equal(durable.sandboxBoundary, 'unresolved'); }); - test('returns a recovered boundary to unresolved after a later sandbox failure', async () => { + test('keeps the boundary unresolved across interleaved successes and a later sandbox failure', async () => { const outcome = await observeFixtureOutcome({ turnEvents: sandboxFailureAfterRecoveryEvents('turn-1'), }); @@ -1562,8 +1614,12 @@ async function* sandboxBoundaryEvents( successToolName = 'Read', ): AsyncIterable { const sameStep = failureStepId !== undefined && failureStepId === successStepId; - if (failureStepId !== undefined) yield toolStart(turnId, 'tool-1', failureStepId, 1); - if (sameStep) yield toolStart(turnId, 'tool-2', successStepId, 2, successToolName); + if (failureStepId !== undefined) { + yield toolStart(turnId, 'tool-1', failureStepId, 1); + } + if (sameStep) { + yield toolStart(turnId, 'tool-2', successStepId, 2, successToolName); + } yield sandboxFailureToolResult(turnId, 3); if (successStepId !== undefined && !sameStep) { yield toolStart(turnId, 'tool-2', successStepId, 4, successToolName); @@ -1572,6 +1628,16 @@ async function* sandboxBoundaryEvents( yield* eventsFor(turnId, text, 6); } +async function* deniedWideningEvents(turnId: string): AsyncIterable { + yield toolStart(turnId, 'tool-1', 'step-1', 1); + yield sandboxFailureToolResult(turnId, 2); + yield toolStart(turnId, 'tool-2', 'step-2', 3, 'request_sandbox_boundary'); + yield successfulToolResult(turnId, 4, 'tool-2'); + yield toolStart(turnId, 'tool-3', 'step-3', 5); + yield successfulToolResult(turnId, 6, 'tool-3'); + yield* eventsFor(turnId, 'Recovered answer', 7); +} + async function* projectedSameStepSandboxFailureEvents(turnId: string): AsyncIterable { yield toolStart(turnId, 'tool-1', 'step-1', 1); yield toolStart(turnId, 'tool-2', 'step-1', 2); diff --git a/packages/cli/src/activation-command.ts b/packages/cli/src/activation-command.ts index aa0fce3f51..17511f233b 100644 --- a/packages/cli/src/activation-command.ts +++ b/packages/cli/src/activation-command.ts @@ -588,10 +588,7 @@ export async function runMakaActivationCli( if (invocation?.failure?.class === 'permission_denied') { return finish('blocked', 'permission_denied', undefined, 'grant_permission'); } - if ( - (streamBoundaryFailure && invocation?.sandboxBoundary !== 'recovered') || - invocation?.sandboxBoundary === 'unresolved' - ) { + if (streamBoundaryFailure || invocation?.sandboxBoundary === 'unresolved') { return finish('blocked', 'permission_required', undefined, 'grant_permission'); } if (!invocation) return finish('fatal_failure', 'missing_invocation'); diff --git a/packages/cli/src/run-command-core.ts b/packages/cli/src/run-command-core.ts index 2fba37b35e..c57897fd0d 100644 --- a/packages/cli/src/run-command-core.ts +++ b/packages/cli/src/run-command-core.ts @@ -82,7 +82,7 @@ export interface MakaRunOutcome { status: 'completed' | 'failed'; finalOutput?: string; failure?: { class: string; message?: string }; - sandboxBoundary: 'none' | 'unresolved' | 'recovered'; + sandboxBoundary: 'none' | 'unresolved'; } export interface MakaRunContextInput { @@ -280,8 +280,7 @@ export async function runMakaTextCliCore( } let outcome: MakaRunOutcome | undefined; - let unclassifiedBoundaryFailure = false; - const boundaryFailureInvocationIds = new Set(); + let boundaryFailure = false; let context: MakaRunContext; try { context = await deps.createContext({ @@ -311,13 +310,7 @@ export async function runMakaTextCliCore( ...(parsed.options.hostProfileId ? { hostProfileId: parsed.options.hostProfileId } : {}), ...(parsed.options.projectId ? { projectId: parsed.options.projectId } : {}), runOutcomeObserver: (result) => { - if (result.sandboxBoundary === 'recovered') { - boundaryFailureInvocationIds.delete(result.outcomeId); - unclassifiedBoundaryFailure = false; - } else if (result.sandboxBoundary === 'unresolved') { - boundaryFailureInvocationIds.add(result.outcomeId); - unclassifiedBoundaryFailure = false; - } + if (result.sandboxBoundary === 'unresolved') boundaryFailure = true; outcome = result; }, }); @@ -403,7 +396,7 @@ export async function runMakaTextCliCore( : {}), })) { if (event.type === 'sandbox_boundary_request') { - unclassifiedBoundaryFailure = true; + boundaryFailure = true; deps.writeStderr( 'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n', ); @@ -414,7 +407,7 @@ export async function runMakaTextCliCore( } const sandboxFailureReason = sessionEventSandboxBoundaryFailureReason(event); if (sandboxFailureReason) { - unclassifiedBoundaryFailure = true; + boundaryFailure = true; deps.writeStderr( sandboxFailureReason === 'requires_bypass' ? 'maka run: sandbox bypass requires an explicit --yolo\n' @@ -446,9 +439,7 @@ export async function runMakaTextCliCore( return 1; } if (streamFailed) return 1; - if (unclassifiedBoundaryFailure || boundaryFailureInvocationIds.size > 0) { - return 1; - } + if (boundaryFailure) return 1; if (!outcome) { deps.writeStderr('maka run: runtime produced no outcome\n'); return 1; diff --git a/packages/cli/src/runtime-host-run-command.ts b/packages/cli/src/runtime-host-run-command.ts index daa99e57b4..9bb0da7e93 100644 --- a/packages/cli/src/runtime-host-run-command.ts +++ b/packages/cli/src/runtime-host-run-command.ts @@ -574,12 +574,6 @@ type TurnOutcomeObservation = readonly status: 'failed'; readonly failure: NonNullable; } - | { - readonly kind: 'tool_call'; - readonly toolUseId: string; - readonly stepId: string | undefined; - readonly toolName: string; - } | { readonly kind: 'tool_result'; readonly toolUseId: string; @@ -590,20 +584,9 @@ type TerminalOutcomeObservation = Extract(); - readonly #unresolvedSandboxFailures = new Map< - string, - { - readonly failedStepId: string | undefined; - readonly failedToolName: string | undefined; - } - >(); + readonly #unresolvedSandboxFailures = new Set(); #finalOutput: string | undefined; #terminal: TerminalOutcomeObservation | undefined; - #sandboxBoundaryRecovered = false; constructor(outcomeId: string) { this.#outcomeId = outcomeId; @@ -621,36 +604,13 @@ class TurnOutcomeClassifier { this.#terminal = observation; } return; - case 'tool_call': - this.#callByToolUseId.set(observation.toolUseId, { - stepId: observation.stepId, - toolName: observation.toolName, - }); - return; case 'tool_result': { - const call = this.#callByToolUseId.get(observation.toolUseId); if (observation.outcome === 'sandbox_failure') { - this.#unresolvedSandboxFailures.set(observation.toolUseId, { - failedStepId: call?.stepId, - failedToolName: call?.toolName, - }); - return; - } - const unresolved = [...this.#unresolvedSandboxFailures.values()]; - // The wire has no retry identity. A later success can only prove recovery - // when there is exactly one unresolved candidate. - if ( - observation.outcome === 'success' && - call?.toolName !== 'request_sandbox_boundary' && - unresolved.length === 1 && - call?.stepId !== undefined && - unresolved[0]?.failedStepId !== undefined && - call.stepId !== unresolved[0].failedStepId && - call.toolName === unresolved[0].failedToolName - ) { - this.#unresolvedSandboxFailures.clear(); - this.#sandboxBoundaryRecovered = true; + this.#unresolvedSandboxFailures.add(observation.toolUseId); } + // No clearing path: `maka run` denies every widening request, so the + // boundary cannot move mid-Turn and a later success cannot prove that + // a blocked call recovered. The failure stays unresolved to the end. return; } } @@ -662,12 +622,7 @@ class TurnOutcomeClassifier { const terminal = this.#terminal; if (!terminal && incomplete === 'pending') return undefined; const completed = terminal?.status === 'completed'; - const sandboxBoundary = - this.#unresolvedSandboxFailures.size > 0 - ? 'unresolved' - : this.#sandboxBoundaryRecovered - ? 'recovered' - : 'none'; + const sandboxBoundary = this.#unresolvedSandboxFailures.size > 0 ? 'unresolved' : 'none'; const failure = terminal?.status === 'failed' ? terminal.failure @@ -708,14 +663,6 @@ function observationFromSessionEvent(event: SessionEvent): TurnOutcomeObservatio if (event.type === 'complete') { return observationFromCompleteEvent(event); } - if (event.type === 'tool_start') { - return { - kind: 'tool_call', - toolUseId: event.toolUseId, - stepId: event.stepId, - toolName: event.toolName, - }; - } return event.type === 'tool_result' ? observationFromToolResult(event) : undefined; } @@ -745,14 +692,6 @@ function observationFromStoredMessage(message: StoredMessage): TurnOutcomeObserv }, }; } - if (message.type === 'tool_call') { - return { - kind: 'tool_call', - toolUseId: message.id, - stepId: message.stepId, - toolName: message.toolName, - }; - } return message.type === 'tool_result' ? observationFromToolResult(message) : undefined; }