From a3f7adbe0dfc3e01c9f7d8bebe6c53ebc9c95870 Mon Sep 17 00:00:00 2001 From: Nick <209239467+surlymochan@users.noreply.github.com> Date: Thu, 12 Nov 2026 09:22:56 +0000 Subject: [PATCH] test(stream): replay 3 real codex runs + acceptance (M14 slice 2, #187) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add 3 verbatim codex --json recordings (gocc, codex-cli 0.166.0, ChatGPT login, 2026-11-12): 2 completed runs (ls, cat) + 1 run killed mid-call (sleep 45, SIGTERM/SIGKILL, exit 143). - New tests: clean replay of all 3 (checkPairing 0 violations, every tool_start paired exactly once), killed run closes as abandoned, acceptance 2 (every call terminal after replay + close). - README provenance updated with recording method + flag note (codex 0.166 dropped --full-auto; sandbox danger-full-access + --dangerously-bypass-approvals-and-sandbox is its equivalent). Partial slice: no real claude stream-json recording — the claude CLI on gocc reports 'Not logged in' over SSH and from the GUI-session bridge alike (OAuth token in macOS keychain, unreachable from any non-interactive context); MacBook has no claude CLI. The claude fixtures stay labelled reconstructed. Refs #187. --- test/agent-stream.test.js | 89 +++++++++++++++++++ test/fixtures/agent-stream/README.md | 46 +++++++++- .../codex-real-killed-sleep.ndjson | 4 + .../agent-stream/codex-real-normal-ls.ndjson | 9 ++ .../codex-real-normal-read.ndjson | 7 ++ 5 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 test/fixtures/agent-stream/codex-real-killed-sleep.ndjson create mode 100644 test/fixtures/agent-stream/codex-real-normal-ls.ndjson create mode 100644 test/fixtures/agent-stream/codex-real-normal-read.ndjson diff --git a/test/agent-stream.test.js b/test/agent-stream.test.js index 0e637d7..d7a7e7d 100644 --- a/test/agent-stream.test.js +++ b/test/agent-stream.test.js @@ -18,6 +18,10 @@ const codexFixture = readFileSync(join(FIXTURES, 'codex.ndjson'), 'utf8'); const claudeFixture = readFileSync(join(FIXTURES, 'claude.ndjson'), 'utf8'); const claudeErrorKilledFixture = readFileSync(join(FIXTURES, 'claude-error-killed.ndjson'), 'utf8'); const codexKilledFixture = readFileSync(join(FIXTURES, 'codex-killed.ndjson'), 'utf8'); +// M14 slice 2 (#187): verbatim recordings from real codex runs (gocc, 2026-11-12). +const codexRealNormalLsFixture = readFileSync(join(FIXTURES, 'codex-real-normal-ls.ndjson'), 'utf8'); +const codexRealKilledSleepFixture = readFileSync(join(FIXTURES, 'codex-real-killed-sleep.ndjson'), 'utf8'); +const codexRealNormalReadFixture = readFileSync(join(FIXTURES, 'codex-real-normal-read.ndjson'), 'utf8'); function makeProjectRoot() { const root = mkdtempSync(join(tmpdir(), 'openflow-agent-stream-')); @@ -37,6 +41,17 @@ function collect(fixture, adapter, options = {}) { return events; } +/** Terminal state of every id-bearing call in a replayed log, or null when a call has none. */ +function terminalStates(events) { + const states = new Map(); + for (const event of events) { + if (!event.tool_call_id) continue; + if (event.event === 'tool_start') states.set(event.tool_call_id, { tool: event.tool, status: 'running' }); + if (event.event === 'tool_end') states.set(event.tool_call_id, { tool: event.tool, status: event.status }); + } + return states; +} + /** Every id-bearing tool_start has exactly one tool_end with the same id. */ function assertPaired(events) { const starts = new Map(); @@ -516,4 +531,78 @@ describe('Agent stream', () => { } }); }); + + // M14 slice 2 (#187): verbatim recordings from real codex runs (gocc, 2026-11-12, + // codex-cli 0.166.0, ChatGPT login). The claude half is still open: no logged-in claude + // CLI is reachable from this worker (SSH or GUI-bridge context both report "Not logged + // in"), so no claude fixture here claims to be a real recording. + describe('real recorded codex streams', () => { + const realRuns = [ + { name: 'codex-real-normal-ls', fixture: codexRealNormalLsFixture, calls: 2, killed: false }, + { name: 'codex-real-normal-read', fixture: codexRealNormalReadFixture, calls: 1, killed: false }, + { name: 'codex-real-killed-sleep', fixture: codexRealKilledSleepFixture, calls: 1, killed: true }, + ]; + + test('replays three real codex runs with zero pairing violations', async () => { + for (const run of realRuns) { + const events = collect(run.fixture, 'codex_cli'); + assert.deepEqual(checkPairing(events), [], `${run.name}: clean replay`); + const projectRoot = makeProjectRoot(); + try { + const { sink, stats, events: logged } = await replayFixture(projectRoot, run.fixture, 'codex_cli', { agentRunId: 'agent_run_test' }); + assert.equal(stats.pairing_violations, 0, `${run.name}: no sink violations`); + assert.deepEqual(sink.pairingViolations, []); + assertPaired(logged); + assert.equal( + logged.filter((entry) => entry.event === 'tool_start').length, + run.calls, + `${run.name}: tool call count`, + ); + } finally { + rmSync(projectRoot, { recursive: true, force: true }); + } + } + }); + + test('the killed real run closes its open call as abandoned', async () => { + const projectRoot = makeProjectRoot(); + try { + const { stats, events } = await replayFixture(projectRoot, codexRealKilledSleepFixture, 'codex_cli', { agentRunId: 'agent_run_test' }); + assert.equal(stats.pairing_violations, 0); + assert.equal(stats.events, 4, 'agent_start + message_end + tool_start + abandoned end'); + assert.deepEqual(events.map((entry) => entry.event), ['agent_start', 'message_end', 'tool_start', 'tool_end']); + assert.equal(events[2].tool, 'shell'); + assert.equal(events[2].tool_call_id, 'item_1'); + assert.equal(events[3].status, 'abandoned'); + assert.equal(events[3].tool_call_id, 'item_1'); + } finally { + rmSync(projectRoot, { recursive: true, force: true }); + } + }); + + test('acceptance 2: after replay + close, every call shows a terminal state', async () => { + for (const run of realRuns) { + const projectRoot = makeProjectRoot(); + try { + const { events } = await replayFixture(projectRoot, run.fixture, 'codex_cli', { agentRunId: 'agent_run_test' }); + const states = terminalStates(events); + assert.equal(states.size, run.calls, `${run.name}: one terminal row per call`); + for (const [callId, state] of states) { + assert.ok( + ['completed', 'error', 'abandoned'].includes(state.status), + `${run.name}: ${callId} left ${state.status}, not terminal`, + ); + } + if (run.killed) { + assert.equal([...states.values()][0].status, 'abandoned'); + } else { + assert.ok([...states.values()].every((state) => state.status === 'completed')); + assert.ok(events.some((entry) => entry.event === 'agent_end'), `${run.name}: agent_end present`); + } + } finally { + rmSync(projectRoot, { recursive: true, force: true }); + } + } + }); + }); }); diff --git a/test/fixtures/agent-stream/README.md b/test/fixtures/agent-stream/README.md index 0db05d7..1a5bbd3 100644 --- a/test/fixtures/agent-stream/README.md +++ b/test/fixtures/agent-stream/README.md @@ -21,6 +21,46 @@ from the real stream formats above and from the CLIs' documented stream-json / `item.id`; a killed run ends after `item.started` with no `item.completed` and no `turn.completed`. -Slice 2 (#184 acceptance 1) replaces / extends these with ≥3 streams recorded -from real CLI runs; until then no test in this directory claims a fixture is a -verbatim recording. +Slice 2 (#187, #184 acceptance 1) adds the three real codex recordings below; +the claude half is still open (see bottom), so the claude fixtures above stay +labelled reconstructed and no test claims a claude fixture is a verbatim +recording. + +`codex-real-normal-ls.ndjson` / `codex-real-normal-read.ndjson` / +`codex-real-killed-sleep.ndjson` (M14 slice 2, #187): **verbatim recordings +from real codex runs** — raw stdout, nothing edited. Recorded 2026-11-12 on +gocc (the user's Mac mini) with codex-cli 0.166.0, logged in using ChatGPT, +in an empty scratch directory (`/tmp/m14s2/work`, one `sample.txt` file). +Commands (stdin from /dev/null): + +- normal-ls: `codex exec --json --skip-git-repo-check -C /tmp/m14s2/work + --sandbox danger-full-access --dangerously-bypass-approvals-and-sandbox + "List the files in this directory with ls, then report how many files + there are. Do not modify anything."` — exit 0, 9 lines, 2 shell calls + (`ls`, `find … | wc -l`), both paired, `turn.completed` present. +- normal-read: same flags, prompt `"Read the file sample.txt in this + directory and report its exact contents. Do not modify anything."` — + exit 0, 7 lines, 1 shell call (`cat sample.txt`), paired, + `turn.completed` present. +- killed-sleep: same flags, prompt `"Run sleep 45 with the shell, then + report when it finishes. Do not modify anything."` — the process was sent + SIGTERM once `item.started` for the `sleep 45` call was observed in the + stream (+5 s), then SIGKILL after 3 s; exit 143. 4 lines: the `sleep 45` + call has an `item.started` and no `item.completed`, and there is no + `turn.completed`. Replaying it through the sink closes that call as + `abandoned`. + +Flag note: codex 0.166.0 dropped `--full-auto` (the flag +`agent-invoke` passes on older codex); `--sandbox danger-full-access +--dangerously-bypass-approvals-and-sandbox` is its documented equivalent +in this version, and the `--json` line shapes are unchanged. + +Claude half still open: no claude stream-json recording is in this slice. +The claude CLI on gocc (2.1.302) reports "Not logged in · Please run +/login" both over SSH and when run from the GUI-session bridge (login +shell included) — its OAuth token lives in the macOS keychain and is not +reachable from any non-interactive context available to this worker, and +the MacBook has no claude CLI at all. A real claude recording needs the +user to run it in their own terminal (see #187 for the exact commands); +until then the claude fixtures above stay labelled reconstructed and no +test claims a claude fixture is a verbatim recording. diff --git a/test/fixtures/agent-stream/codex-real-killed-sleep.ndjson b/test/fixtures/agent-stream/codex-real-killed-sleep.ndjson new file mode 100644 index 0000000..6d900b6 --- /dev/null +++ b/test/fixtures/agent-stream/codex-real-killed-sleep.ndjson @@ -0,0 +1,4 @@ +{"type":"thread.started","thread_id":"01a1cf05-fc05-7162-98af-8de68657b67f"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll run `sleep 45` in the shell and report when it finishes."}} +{"type":"item.started","item":{"id":"item_1","type":"command_execution","command":"/bin/zsh -lc 'sleep 45'","aggregated_output":"","exit_code":null,"status":"in_progress"}} diff --git a/test/fixtures/agent-stream/codex-real-normal-ls.ndjson b/test/fixtures/agent-stream/codex-real-normal-ls.ndjson new file mode 100644 index 0000000..b18ef8c --- /dev/null +++ b/test/fixtures/agent-stream/codex-real-normal-ls.ndjson @@ -0,0 +1,9 @@ +{"type":"thread.started","thread_id":"01a1cf05-073a-7370-bbf4-d18a84006e81"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll run `ls` and count the files without modifying anything.\n"}} +{"type":"item.started","item":{"id":"item_1","type":"command_execution","command":"/bin/zsh -lc ls","aggregated_output":"","exit_code":null,"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"/bin/zsh -lc ls","aggregated_output":"sample.txt\n","exit_code":0,"status":"completed"}} +{"type":"item.started","item":{"id":"item_2","type":"command_execution","command":"/bin/zsh -lc 'find . -maxdepth 1 -type f | wc -l'","aggregated_output":"","exit_code":null,"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_2","type":"command_execution","command":"/bin/zsh -lc 'find . -maxdepth 1 -type f | wc -l'","aggregated_output":" 1\n","exit_code":0,"status":"completed"}} +{"type":"item.completed","item":{"id":"item_3","type":"agent_message","text":"`ls` listed `sample.txt`.\n\nThere is **1 file**. Nothing was modified."}} +{"type":"turn.completed","usage":{"input_tokens":52943,"cached_input_tokens":33408,"cache_write_input_tokens":0,"output_tokens":123,"reasoning_output_tokens":0}} diff --git a/test/fixtures/agent-stream/codex-real-normal-read.ndjson b/test/fixtures/agent-stream/codex-real-normal-read.ndjson new file mode 100644 index 0000000..55f56b5 --- /dev/null +++ b/test/fixtures/agent-stream/codex-real-normal-read.ndjson @@ -0,0 +1,7 @@ +{"type":"thread.started","thread_id":"01a1cf09-9cd0-7670-a4a2-481104268e69"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"I’ll read `sample.txt` without changing anything."}} +{"type":"item.started","item":{"id":"item_1","type":"command_execution","command":"/bin/zsh -lc 'cat sample.txt'","aggregated_output":"","exit_code":null,"status":"in_progress"}} +{"type":"item.completed","item":{"id":"item_1","type":"command_execution","command":"/bin/zsh -lc 'cat sample.txt'","aggregated_output":"hello\n","exit_code":0,"status":"completed"}} +{"type":"item.completed","item":{"id":"item_2","type":"agent_message","text":"```text\nhello\n```\n\nThe file ends with a newline."}} +{"type":"turn.completed","usage":{"input_tokens":53266,"cached_input_tokens":39680,"cache_write_input_tokens":0,"output_tokens":76,"reasoning_output_tokens":0}}