Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions packages/cli/src/__tests__/activation-command.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,59 @@ describe('maka activate JSONL protocol', () => {
});
});

test('blocks a completed invocation whose stream carried a boundary failure', async () => {
// Guards the current contract: a completed invocation whose stream carried
// a boundary failure reports `blocked` / `permission_required` with exit 3.
// The `maka activate` transition itself (main completed with exit 0 when
// the classifier cleared `recovered`) is not regression-coverable after
// the deletion: `recovered` no longer exists in the outcome type, so an
// injected `MakaRunOutcome` cannot express the old shape.
const lines: string[] = [];
const boundaryFailure = {
kind: 'text',
text: 'Write requires an approved session sandbox boundary expansion.',
sandboxFailure: {
reason: 'sandbox_boundary_required',
requiredExpansion: {
filesystem: {
entries: [{ path: '/tmp/output', access: 'write', scope: 'subtree' }],
},
},
},
} as const;
const result = await runMakaActivationCli(
[
'--state-root',
ROOTS.stateRoot,
'--workspace-root',
ROOTS.workspaceRoot,
'--config-root',
ROOTS.configRoot,
],
{
...fakeDeps({
result: completedResult(),
events: [
{
type: 'tool_result',
id: 'event-boundary-result',
turnId: 'turn-1',
ts: 1,
toolUseId: 'tool-boundary',
isError: true,
content: boundaryFailure,
},
],
}),
writeStdout: (text) => lines.push(text.trim()),
},
);

assert.equal(result, 3);
assert.equal(JSON.parse(lines.at(-1)!).status, 'blocked');
assert.equal(JSON.parse(lines.at(-1)!).reason, 'permission_required');
});

test('retries non-permission blocked sessions instead of requesting permission', async () => {
for (const blockedReason of ['auth', 'tool_failed'] as const) {
const lines: string[] = [];
Expand Down
88 changes: 77 additions & 11 deletions packages/cli/src/__tests__/runtime-host-run-command.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -313,6 +313,50 @@ describe('Runtime Host maka run adapter', () => {
);
});

test('keeps the boundary unresolved when a later same-named call succeeds (live)', async () => {
const fixture = runFixture({
turnEvents: sandboxBoundaryEvents('turn-1', 'step-1', 'step-2', 'Partial answer', 'Read'),
});

const exitCode = await runFixtureCommand(fixture, ['accept same-name different target']);

assert.equal(exitCode, 1);
});

test('keeps the boundary unresolved when a later same-named call succeeds (durable)', async () => {
const fixture = runFixture({
graph: true,
finalMessages: sandboxBoundaryMessages('step-1', 'step-2', 'Read'),
});

const exitCode = await runFixtureCommand(fixture, [
'accept durable same-name different target',
'--graph',
]);

assert.equal(exitCode, 1);
});

test('returns exit code 1 when a denied widening precedes any later tool success', async () => {
const stderr: string[] = [];
const fixture = runFixture({
turnEvents: deniedWideningEvents('turn-1'),
});

const exitCode = await runFixtureCommand(
fixture,
['accept post-denial success'],
() => {},
(text) => stderr.push(text),
);

assert.equal(exitCode, 1);
assert.equal(
stderr.join(''),
'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n',
);
});

test('returns exit code 1 when reconnect restores a missed sandbox failure', async () => {
let publishReplacement = () => {};
const fixture = runFixture({
Expand Down Expand Up @@ -351,18 +395,26 @@ describe('Runtime Host maka run adapter', () => {
assert.equal(stdout.join(''), 'Final graph answer\n');
});

test('returns exit code 0 when a root Graph boundary failure recovers', async () => {
test('keeps a root Graph boundary failure unresolved even when a later same-named call succeeds', async () => {
const stdout: string[] = [];
const stderr: string[] = [];
const fixture = runFixture({
graph: true,
turnEvents: sandboxBoundaryEvents('turn-1', 'step-1', 'step-2', 'Recovered answer'),
});
const exitCode = await runFixtureCommand(fixture, ['recover once', '--graph'], (text) =>
stdout.push(text),
const exitCode = await runFixtureCommand(
fixture,
['recover once', '--graph'],
(text) => stdout.push(text),
(text) => stderr.push(text),
);

assert.equal(exitCode, 0);
assert.equal(stdout.join(''), 'Final graph answer\n');
assert.equal(exitCode, 1);
assert.equal(stdout.join(''), '');
assert.equal(
stderr.join(''),
'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n',
);
});

test('returns exit code 1 when a same-step Graph sibling succeeds after a sandbox failure', async () => {
Expand Down Expand Up @@ -465,7 +517,7 @@ describe('Runtime Host maka run adapter', () => {
assert.equal(observed.at(-1)?.finalOutput, 'Final graph answer');
});

test('reports a recovered sandbox boundary from live and durable Turns', async () => {
test('keeps the sandbox boundary unresolved across live and durable Turns', async () => {
const live = await observeFixtureOutcome({
turnEvents: sandboxBoundaryEvents('turn-1', 'step-1', 'step-2', 'Recovered answer'),
});
Expand All @@ -474,8 +526,8 @@ describe('Runtime Host maka run adapter', () => {
finalMessages: sandboxBoundaryMessages('step-1', 'step-2'),
});

assert.equal(live.sandboxBoundary, 'recovered');
assert.equal(durable.sandboxBoundary, 'recovered');
assert.equal(live.sandboxBoundary, 'unresolved');
assert.equal(durable.sandboxBoundary, 'unresolved');
});

test('leaves sandbox failures unresolved when their provider steps are unavailable', async () => {
Expand All @@ -491,7 +543,7 @@ describe('Runtime Host maka run adapter', () => {
assert.equal(durable.sandboxBoundary, 'unresolved');
});

test('returns a recovered boundary to unresolved after a later sandbox failure', async () => {
test('keeps the boundary unresolved across interleaved successes and a later sandbox failure', async () => {
const outcome = await observeFixtureOutcome({
turnEvents: sandboxFailureAfterRecoveryEvents('turn-1'),
});
Expand Down Expand Up @@ -1562,8 +1614,12 @@ async function* sandboxBoundaryEvents(
successToolName = 'Read',
): AsyncIterable<SessionEvent> {
const sameStep = failureStepId !== undefined && failureStepId === successStepId;
if (failureStepId !== undefined) yield toolStart(turnId, 'tool-1', failureStepId, 1);
if (sameStep) yield toolStart(turnId, 'tool-2', successStepId, 2, successToolName);
if (failureStepId !== undefined) {
yield toolStart(turnId, 'tool-1', failureStepId, 1);
}
if (sameStep) {
yield toolStart(turnId, 'tool-2', successStepId, 2, successToolName);
}
yield sandboxFailureToolResult(turnId, 3);
if (successStepId !== undefined && !sameStep) {
yield toolStart(turnId, 'tool-2', successStepId, 4, successToolName);
Expand All @@ -1572,6 +1628,16 @@ async function* sandboxBoundaryEvents(
yield* eventsFor(turnId, text, 6);
}

async function* deniedWideningEvents(turnId: string): AsyncIterable<SessionEvent> {
yield toolStart(turnId, 'tool-1', 'step-1', 1);
yield sandboxFailureToolResult(turnId, 2);
yield toolStart(turnId, 'tool-2', 'step-2', 3, 'request_sandbox_boundary');
yield successfulToolResult(turnId, 4, 'tool-2');
yield toolStart(turnId, 'tool-3', 'step-3', 5);
yield successfulToolResult(turnId, 6, 'tool-3');
yield* eventsFor(turnId, 'Recovered answer', 7);
}

async function* projectedSameStepSandboxFailureEvents(turnId: string): AsyncIterable<SessionEvent> {
yield toolStart(turnId, 'tool-1', 'step-1', 1);
yield toolStart(turnId, 'tool-2', 'step-1', 2);
Expand Down
5 changes: 1 addition & 4 deletions packages/cli/src/activation-command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -588,10 +588,7 @@ export async function runMakaActivationCli(
if (invocation?.failure?.class === 'permission_denied') {
return finish('blocked', 'permission_denied', undefined, 'grant_permission');
}
if (
(streamBoundaryFailure && invocation?.sandboxBoundary !== 'recovered') ||
invocation?.sandboxBoundary === 'unresolved'
) {
if (streamBoundaryFailure || invocation?.sandboxBoundary === 'unresolved') {
return finish('blocked', 'permission_required', undefined, 'grant_permission');
}
if (!invocation) return finish('fatal_failure', 'missing_invocation');
Expand Down
21 changes: 6 additions & 15 deletions packages/cli/src/run-command-core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ export interface MakaRunOutcome {
status: 'completed' | 'failed';
finalOutput?: string;
failure?: { class: string; message?: string };
sandboxBoundary: 'none' | 'unresolved' | 'recovered';
sandboxBoundary: 'none' | 'unresolved';
}

export interface MakaRunContextInput {
Expand Down Expand Up @@ -280,8 +280,7 @@ export async function runMakaTextCliCore(
}

let outcome: MakaRunOutcome | undefined;
let unclassifiedBoundaryFailure = false;
const boundaryFailureInvocationIds = new Set<string>();
let boundaryFailure = false;
let context: MakaRunContext;
try {
context = await deps.createContext({
Expand Down Expand Up @@ -311,13 +310,7 @@ export async function runMakaTextCliCore(
...(parsed.options.hostProfileId ? { hostProfileId: parsed.options.hostProfileId } : {}),
...(parsed.options.projectId ? { projectId: parsed.options.projectId } : {}),
runOutcomeObserver: (result) => {
if (result.sandboxBoundary === 'recovered') {
boundaryFailureInvocationIds.delete(result.outcomeId);
unclassifiedBoundaryFailure = false;
} else if (result.sandboxBoundary === 'unresolved') {
boundaryFailureInvocationIds.add(result.outcomeId);
unclassifiedBoundaryFailure = false;
}
if (result.sandboxBoundary === 'unresolved') boundaryFailure = true;
outcome = result;
},
});
Expand Down Expand Up @@ -403,7 +396,7 @@ export async function runMakaTextCliCore(
: {}),
})) {
if (event.type === 'sandbox_boundary_request') {
unclassifiedBoundaryFailure = true;
boundaryFailure = true;
deps.writeStderr(
'maka run: sandbox boundary expansion is unavailable in non-interactive mode\n',
);
Expand All @@ -414,7 +407,7 @@ export async function runMakaTextCliCore(
}
const sandboxFailureReason = sessionEventSandboxBoundaryFailureReason(event);
if (sandboxFailureReason) {
unclassifiedBoundaryFailure = true;
boundaryFailure = true;
deps.writeStderr(
sandboxFailureReason === 'requires_bypass'
? 'maka run: sandbox bypass requires an explicit --yolo\n'
Expand Down Expand Up @@ -446,9 +439,7 @@ export async function runMakaTextCliCore(
return 1;
}
if (streamFailed) return 1;
if (unclassifiedBoundaryFailure || boundaryFailureInvocationIds.size > 0) {
return 1;
}
if (boundaryFailure) return 1;
if (!outcome) {
deps.writeStderr('maka run: runtime produced no outcome\n');
return 1;
Expand Down
73 changes: 6 additions & 67 deletions packages/cli/src/runtime-host-run-command.ts
Original file line number Diff line number Diff line change
Expand Up @@ -574,12 +574,6 @@ type TurnOutcomeObservation =
readonly status: 'failed';
readonly failure: NonNullable<MakaRunOutcome['failure']>;
}
| {
readonly kind: 'tool_call';
readonly toolUseId: string;
readonly stepId: string | undefined;
readonly toolName: string;
}
| {
readonly kind: 'tool_result';
readonly toolUseId: string;
Expand All @@ -590,20 +584,9 @@ type TerminalOutcomeObservation = Extract<TurnOutcomeObservation, { kind: 'termi

class TurnOutcomeClassifier {
readonly #outcomeId: string;
readonly #callByToolUseId = new Map<
string,
{ readonly stepId: string | undefined; readonly toolName: string }
>();
readonly #unresolvedSandboxFailures = new Map<
string,
{
readonly failedStepId: string | undefined;
readonly failedToolName: string | undefined;
}
>();
readonly #unresolvedSandboxFailures = new Set<string>();
#finalOutput: string | undefined;
#terminal: TerminalOutcomeObservation | undefined;
#sandboxBoundaryRecovered = false;

constructor(outcomeId: string) {
this.#outcomeId = outcomeId;
Expand All @@ -621,36 +604,13 @@ class TurnOutcomeClassifier {
this.#terminal = observation;
}
return;
case 'tool_call':
this.#callByToolUseId.set(observation.toolUseId, {
stepId: observation.stepId,
toolName: observation.toolName,
});
return;
case 'tool_result': {
const call = this.#callByToolUseId.get(observation.toolUseId);
if (observation.outcome === 'sandbox_failure') {
this.#unresolvedSandboxFailures.set(observation.toolUseId, {
failedStepId: call?.stepId,
failedToolName: call?.toolName,
});
return;
}
const unresolved = [...this.#unresolvedSandboxFailures.values()];
// The wire has no retry identity. A later success can only prove recovery
// when there is exactly one unresolved candidate.
if (
observation.outcome === 'success' &&
call?.toolName !== 'request_sandbox_boundary' &&
unresolved.length === 1 &&
call?.stepId !== undefined &&
unresolved[0]?.failedStepId !== undefined &&
call.stepId !== unresolved[0].failedStepId &&
call.toolName === unresolved[0].failedToolName
) {
this.#unresolvedSandboxFailures.clear();
this.#sandboxBoundaryRecovered = true;
this.#unresolvedSandboxFailures.add(observation.toolUseId);
}
// No clearing path: `maka run` denies every widening request, so the
// boundary cannot move mid-Turn and a later success cannot prove that
// a blocked call recovered. The failure stays unresolved to the end.
return;
}
}
Expand All @@ -662,12 +622,7 @@ class TurnOutcomeClassifier {
const terminal = this.#terminal;
if (!terminal && incomplete === 'pending') return undefined;
const completed = terminal?.status === 'completed';
const sandboxBoundary =
this.#unresolvedSandboxFailures.size > 0
? 'unresolved'
: this.#sandboxBoundaryRecovered
? 'recovered'
: 'none';
const sandboxBoundary = this.#unresolvedSandboxFailures.size > 0 ? 'unresolved' : 'none';
const failure =
terminal?.status === 'failed'
? terminal.failure
Expand Down Expand Up @@ -708,14 +663,6 @@ function observationFromSessionEvent(event: SessionEvent): TurnOutcomeObservatio
if (event.type === 'complete') {
return observationFromCompleteEvent(event);
}
if (event.type === 'tool_start') {
return {
kind: 'tool_call',
toolUseId: event.toolUseId,
stepId: event.stepId,
toolName: event.toolName,
};
}
return event.type === 'tool_result' ? observationFromToolResult(event) : undefined;
}

Expand Down Expand Up @@ -745,14 +692,6 @@ function observationFromStoredMessage(message: StoredMessage): TurnOutcomeObserv
},
};
}
if (message.type === 'tool_call') {
return {
kind: 'tool_call',
toolUseId: message.id,
stepId: message.stepId,
toolName: message.toolName,
};
}
return message.type === 'tool_result' ? observationFromToolResult(message) : undefined;
}

Expand Down
Loading