diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed724e3..0629d6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,3 +35,7 @@ jobs: npm run --silent demo:repair -- --show-stop > "$result" || status=$? test "$status" -eq 1 jq -e '.status == "repair_limit" and (.consultation.attempts | length) == 2' "$result" + - name: Verify the diagnosis path + run: | + npm run --silent demo:repair -- --show-diagnosis > diagnosis.json + jq -e '.status == "passed" and .diagnosis.action == "repair" and [.attempts[].number] == [1, 2, 3]' diagnosis.json diff --git a/README.ja.md b/README.ja.md index e635d3b..be6b3b1 100644 --- a/README.ja.md +++ b/README.ja.md @@ -26,6 +26,8 @@ npm run --silent demo:repair -- --show-stop `npm run --silent demo:repair -- --chatgpt` では、ChatGPT にサインインした Codex が、使い捨ての試料を固定 Low で直します。修正がもう一度必要なら、同じセッションに新しい検査結果だけを足します。Codex が返した場合はキャッシュ済み入力トークンも表示します。[呼び出し元向けの関数](src/repair-loop.mjs)は既定で2回まで修正し、`maxRepairs` に正の整数を指定すると上限を変えられます。 +モデルを呼ばずに見直しの流れを試すには `npm run --silent demo:repair -- --show-diagnosis` を実行します。組み込む際は `diagnose` を渡します。既定では2回の修正後に検査が落ちると、最新の検査、修正の経過、差分が `effort: "high"` とともに渡ります。見直しは読み取り専用で実行し、空でない文字列を持つ `{ action, cause, evidence, next }` を返します。現行仕様内で具体的に直せるなら `repair`、原因が不明なら `unknown`、仕様変更など判断が要るなら `human_decision` を返します。`repair` の場合だけ Low で1回修正し、同じ独立検査で確かめます。モデルの推論設定と読み取り専用の隔離は呼び出し側が適用します。 + ## 検査から始める作り方 1. **再現する。** 何が通ればよいかを決め、異常例と正常例を一つずつ試します。 diff --git a/README.md b/README.md index 96e118d..68df4ec 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,8 @@ This exits with code 1 and returns a `consultation` object with both attempts, t Run `npm run --silent demo:repair -- --chatgpt` to have Codex apply the fix at fixed Low using your ChatGPT sign-in. The demo uses a disposable checkout-free fixture. A follow-up appends only the new check result to the same Codex session. Reported usage includes cached input when Codex supplies it. The [runRepairLoop API](src/repair-loop.mjs) defaults to two repair attempts. Callers can set `maxRepairs` to a positive integer. +To try the diagnosis path without model calls, run `npm run --silent demo:repair -- --show-diagnosis`. Pass a `diagnose` callback to enable this path in your integration. After two failed repairs by default, it receives the latest check, attempts, and diff with `effort: "high"`. Run it with read-only access. Return `{ action, cause, evidence, next }` with nonempty strings. Use `repair` when the evidence supports a concrete change within the current requirements, `unknown` when the cause is unclear, or `human_decision` when approval is needed. Only `repair` permits one more repair at `effort: "low"`, followed by the same independent check. The caller sets the model effort and provides read-only isolation. + ## The workflow 1. **Reproduce.** Run a small check against the behavior you expect, with a normal case to check for regressions. diff --git a/README.zh-CN.md b/README.zh-CN.md index 36cddec..c3da5c1 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -26,6 +26,8 @@ npm run --silent demo:repair -- --show-stop 如已用 ChatGPT 登录 Codex,可以运行 `npm run --silent demo:repair -- --chatgpt`,让 Codex 以固定的 Low 推理强度修复临时示例。需要再次修复时,只向同一个会话追加新的检查结果。若 Codex 提供缓存数据,输出中也会显示缓存的输入 token 数。[调用接口 runRepairLoop](src/repair-loop.mjs)默认最多修复两次,调用方可通过正整数 `maxRepairs` 调整上限。 +运行 `npm run --silent demo:repair -- --show-diagnosis` 可在不调用模型的情况下试用诊断流程。接入时提供 `diagnose` 回调。默认两次修复后检查仍失败,回调将收到最新检查结果、修复记录、差异和 `effort: "high"`。请以只读方式运行诊断,返回各字段均为非空字符串的 `{ action, cause, evidence, next }`。有证据支持在现有需求内修复时返回 `repair`,原因不明确时返回 `unknown`,需要人工决定时返回 `human_decision`。只有 `repair` 会触发一次 Low 修复,然后重新运行相同的独立检查。调用方负责设置模型推理强度和只读隔离。 + ## 工作流程 1. **复现问题。** 用一个小测试锁定预期行为,同时保留正常对照用例。 diff --git a/examples/repair-loop.mjs b/examples/repair-loop.mjs index 25c02a9..d02de9b 100644 --- a/examples/repair-loop.mjs +++ b/examples/repair-loop.mjs @@ -11,9 +11,12 @@ import { runRepairLoop } from '../src/repair-loop.mjs'; const { values } = parseArgs({ options: { chatgpt: { type: 'boolean' }, 'show-stop': { type: 'boolean' }, + 'show-diagnosis': { type: 'boolean' }, model: { type: 'string', default: 'gpt-6.2-sol' }, } }); -if (values.chatgpt && values['show-stop']) throw new Error('Choose the live repair or the scripted stop.'); +if ([values.chatgpt, values['show-stop'], values['show-diagnosis']].filter(Boolean).length > 1) { + throw new Error('Choose one demo mode.'); +} const workspace = await mkdtemp(join(tmpdir(), 'test-feedback-repair-')); const implementation = join(workspace, 'answer.mjs'); @@ -100,7 +103,8 @@ try { }; const repair = values.chatgpt ? repairWithCodex : async ({ attempt, evidence }) => { if (!evidence.length) throw new Error('The repair requires assertion evidence'); - const answer = values['show-stop'] ? 42 + attempt : 42; + const simulateMiss = values['show-stop'] || (values['show-diagnosis'] && attempt <= 2); + const answer = simulateMiss ? 42 + attempt : 42; await writeFile(implementation, `export const answer = () => ${answer};\n`); }; const readDiff = async () => { @@ -110,7 +114,13 @@ try { ...original.trimEnd().split('\n').map(line => `-${line}`), ...(current === null ? [] : current.trimEnd().split('\n').map(line => `+${line}`))].join('\n'); }; - const result = await runRepairLoop({ check, repair, readDiff }); + const diagnose = values['show-diagnosis'] ? async ({ latest }) => ({ + action: 'repair', + cause: 'answer.mjs returns the wrong constant.', + evidence: latest.evidence.map(item => item.message).join('\n'), + next: 'Return 42 as required by the original regression test.', + }) : undefined; + const result = await runRepairLoop({ check, repair, readDiff, diagnose }); console.log(JSON.stringify(result, null, 2)); if (result.consultation) process.exitCode = result.status === 'unknown' ? 2 : 1; } finally { diff --git a/src/repair-loop.mjs b/src/repair-loop.mjs index 5354976..8646b7e 100644 --- a/src/repair-loop.mjs +++ b/src/repair-loop.mjs @@ -1,6 +1,11 @@ const stopSummary = { unknown: 'The independent check could not establish a result.', repair_interrupted: 'The repair turn did not complete.', + diagnosis_unknown: 'The diagnosis could not identify an actionable cause.', + diagnosis_needs_decision: 'The diagnosis requires a human decision.', + diagnosis_interrupted: 'The diagnosis did not return a usable result.', + diagnosis_diff_unavailable: 'The project diff could not be checked after diagnosis.', + diagnosis_changed_diff: 'The project diff changed during the read-only diagnosis.', no_change: 'The last repair left the project diff unchanged and the check still failed.', }; @@ -8,27 +13,35 @@ const questions = { repair_limit: 'Do the expected behavior, test inputs, and chosen code boundary still describe the same problem?', unknown: 'Can the same check run to completion in this environment?', repair_interrupted: 'What interrupted the repair, and can the same session safely continue?', + diagnosis_interrupted: 'Can the diagnosis be run again with the same evidence?', + diagnosis_diff_unavailable: 'Can the current project diff be read before retrying?', + diagnosis_changed_diff: 'What changed during diagnosis, and can the original diff be restored?', no_change: 'What prevented a code change, and does the task need a decision before another attempt?', }; /** The caller supplies a trusted check, a repair session, and a project diff reader. */ -export async function runRepairLoop({ check, repair, readDiff, maxRepairs = 2 }) { +export async function runRepairLoop({ check, repair, readDiff, maxRepairs = 2, diagnose }) { if (!Number.isSafeInteger(maxRepairs) || maxRepairs < 1) { throw new RangeError('maxRepairs must be a positive safe integer'); } + if (diagnose !== undefined && typeof diagnose !== 'function') { + throw new TypeError('diagnose must be a function'); + } const attempts = []; + let diagnosis; const initial = await check({ id: 'before' }); let latest = initial; let diff = await readDiff(); function result(status) { - const run = { status, initial, attempts, latest }; + const run = { status, initial, attempts, latest, ...(diagnosis && { diagnosis }) }; if (status === 'passed' || status === 'already_passed') return run; const question = latest.reason === 'regression_test_changed' ? 'Can the original independent check be restored before resuming?' - : questions[status]; + : (status === 'diagnosis_unknown' || status === 'diagnosis_needs_decision' + ? diagnosis.next : questions[status]); const summary = status === 'repair_limit' - ? `The independent check still failed after reaching the repair limit (${maxRepairs}).` + ? `The independent check still failed after reaching the repair limit (${attempts.length}).` : stopSummary[status]; return { ...run, @@ -39,8 +52,10 @@ export async function runRepairLoop({ check, repair, readDiff, maxRepairs = 2 }) if (initial.observation.outcome === 'success') return result('already_passed'); if (initial.observation.outcome !== 'failure') return result('unknown'); - for (let number = 1; number <= maxRepairs; number++) { - const problem = { ...latest, attempt: number }; + for (let number = 1; number <= maxRepairs + (diagnosis?.action === 'repair' ? 1 : 0); number++) { + const problem = diagnose + ? { ...latest, attempt: number, effort: 'low', ...(diagnosis && { diagnosis }) } + : { ...latest, attempt: number }; let usage; try { usage = await repair(problem); @@ -59,8 +74,35 @@ export async function runRepairLoop({ check, repair, readDiff, maxRepairs = 2 }) diff = changed; return result('unknown'); } - if (changed === diff) return result('no_change'); + const hasChange = changed !== diff; diff = changed; + if (diagnose && number === maxRepairs) { + let decision; + try { + decision = await diagnose(structuredClone({ initial, attempts, latest, diff, effort: 'high' })); + } catch { + // Preserve the latest evidence when the diagnostic callback fails. + } + let afterDiagnosis; + try { + afterDiagnosis = await readDiff(); + } catch { + return result('diagnosis_diff_unavailable'); + } + if (afterDiagnosis !== diff) { + diff = afterDiagnosis; + return result('diagnosis_changed_diff'); + } + if (!['repair', 'unknown', 'human_decision'].includes(decision?.action) || + !['cause', 'evidence', 'next'].every(field => typeof decision[field] === 'string' && decision[field].trim())) { + return result('diagnosis_interrupted'); + } + diagnosis = { action: decision.action, cause: decision.cause, evidence: decision.evidence, next: decision.next }; + if (diagnosis.action === 'unknown') return result('diagnosis_unknown'); + if (diagnosis.action === 'human_decision') return result('diagnosis_needs_decision'); + } else if (!hasChange) { + return result('no_change'); + } } return result('repair_limit'); diff --git a/test/repair-loop.test.mjs b/test/repair-loop.test.mjs index 3ffdca1..44f9ee1 100644 --- a/test/repair-loop.test.mjs +++ b/test/repair-loop.test.mjs @@ -25,14 +25,17 @@ test('the second repair receives fresh evidence and a third repair is never star test('an inconclusive recheck stops and preserves its reason and the diff', async () => { let changed = false; + let diagnosed = false; const result = await runRepairLoop({ check: async ({ id }) => changed ? { observation: { id, outcome: 'unknown' }, evidence: [], reason: 'collection_failed' } : { observation: { id, outcome: 'failure' }, evidence: [{ message: 'first failure' }] }, repair: async () => { changed = true; }, readDiff: async () => changed ? 'new diff' : '', + diagnose: async () => { diagnosed = true; }, }); assert.equal(result.status, 'unknown'); + assert.equal(diagnosed, false); assert.equal(result.attempts.length, 1); assert.equal(result.consultation.latest.reason, 'collection_failed'); assert.equal(result.consultation.diff, 'new diff'); @@ -52,3 +55,74 @@ test('callers can set a positive repair limit', async () => { assert.equal(result.status, 'repair_limit'); assert.match(result.consultation.summary, /\(3\)/); }); + +test('two failed Low repairs trigger one High diagnosis and a verified Low repair', async () => { + let edits = 0; + const turns = []; + const result = await runRepairLoop({ + check: async ({ id, resolves }) => ({ + observation: { id, resolves, outcome: edits === 3 ? 'success' : 'failure' }, + evidence: [`observed ${edits}`], + }), + repair: async ({ effort, diagnosis }) => { + turns.push(effort); + if (++edits === 3) assert.equal(diagnosis.next, 'Correct the existing branch.'); + }, + readDiff: async () => `diff ${edits}`, + diagnose: async ({ effort, attempts, latest, diff }) => { + turns.push(effort); + assert.equal(attempts.length, 2); + assert.deepEqual(latest.evidence, ['observed 2']); + assert.equal(diff, 'diff 2'); + return { action: 'repair', cause: 'Wrong branch.', evidence: 'Observed branch output.', next: 'Correct the existing branch.' }; + }, + }); + assert.deepEqual(turns, ['low', 'low', 'high', 'low']); + assert.equal(result.status, 'passed'); + assert.equal(result.latest.observation.resolves, 'after-2'); +}); + +test('an unknown cause, a human decision, or a failed diagnosis stops before any more repairs', async () => { + for (const [action, status] of [ + ['unknown', 'diagnosis_unknown'], ['human_decision', 'diagnosis_needs_decision'], + ['throw', 'diagnosis_interrupted'], ['invalid', 'diagnosis_interrupted'], + ]) { + let edits = 0; + const result = await runRepairLoop({ + check: async ({ id }) => ({ observation: { id, outcome: 'failure' }, evidence: ['mismatch'] }), + repair: async () => { edits++; }, + readDiff: async () => String(edits), + diagnose: async () => { + if (action === 'throw') throw new Error('Model unavailable'); + if (action === 'invalid') return { action: 'repair' }; + return { action, cause: 'Unclear expectation.', evidence: 'Tests disagree.', next: 'Confirm the expected output.' }; + }, + }); + assert.equal(edits, 2); + assert.equal(result.status, status); + assert.equal(result.consultation.diff, '2'); + assert.deepEqual(result.consultation.latest.evidence, ['mismatch']); + if (result.diagnosis) assert.equal(result.consultation.question, result.diagnosis.next); + } +}); + +test('a modified diff blocks repair and a failed post-diagnosis repair cannot loop', async () => { + for (const mutateDuringDiagnosis of [true, false]) { + let state = 0; + let diagnoses = 0; + let repairs = 0; + const result = await runRepairLoop({ + check: async ({ id }) => ({ observation: { id, outcome: 'failure' } }), + repair: async () => { repairs++; state++; }, + readDiff: async () => String(state), + diagnose: async () => { + diagnoses++; + if (mutateDuringDiagnosis) state++; + return { action: 'repair', cause: 'Wrong line.', evidence: 'Assertion failed.', next: 'Correct the line.' }; + }, + }); + assert.equal(diagnoses, 1); + assert.equal(repairs, mutateDuringDiagnosis ? 2 : 3); + assert.equal(result.status, mutateDuringDiagnosis ? 'diagnosis_changed_diff' : 'repair_limit'); + } +});