refactor(evaluation-system): compact harness prompts, A/B verified

plugins/strapped/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "strapped",
-  "version": "0.8.0",
+  "version": "0.8.1",
   "description": "Adversarial plan → implement → stacked-PR coding harness: rule-partitioned reviewers, refute passes, DAG deliverables in persistent worktrees, CLAUDE.md learning loop",
   "author": {
     "name": "Christian Schuetz",

plugins/strapped/workflows/strapped-run.js

@@ -726,7 +726,7 @@ var PR_SCHEMA = {
 // src/workflows/strapped-run/review-loop.ts
 var PLAN_LENSES = {
   a: "completeness: is every element of the original ask covered by some deliverable? Hunt for missing requirements, unhandled edge cases, acceptance criteria without tests, and parts of the ask that silently disappeared",
-  b: "soundness: wrong assumptions about the codebase, DAG dependency errors (missing or backwards deps, undeclared cross-deliverable coupling), deliverables that mix unrelated themes or whose estimated meaningful diff (excluding generated code, dependency bumps, and fixtures) exceeds ~1,000 lines and should be split, deliverables/chains that should be CONSOLIDATED (fragments of one theme, or a linear chain whose combined meaningful diff — excluding generated code, dependency bumps, and fixtures — is under the ~1,000-line threshold and could be a single deliverable/PR), planned work that is dead, duplicated, or superseded within the plan (steps or files a later step obviates, two deliverables doing the same work, or acceptance criteria/tests no remaining step produces), and steps that cannot work as written"
+  b: "soundness: wrong assumptions about the codebase, DAG dependency errors (missing or backwards deps, undeclared cross-deliverable coupling), and steps that cannot work as written. Meaningful diff = changed lines excluding generated code, dependency bumps, and fixtures: SPLIT any deliverable that mixes unrelated themes or whose meaningful diff exceeds ~1,000 lines, and CONSOLIDATE fragments of one theme (or a linear chain whose combined meaningful diff is under that threshold) into one deliverable/PR. Flag planned work that is dead, duplicated, or superseded within the plan — steps or files a later step obviates, two deliverables doing the same work, or acceptance criteria/tests no remaining step produces."
 };
 function ruleBlock(rules) {
   return rules.map((r) => `- ${r.id} (${r.source}): ${r.text}`).join(`
@@ -762,7 +762,7 @@ ${ruleBlock(rules)}
 Known findings from earlier rounds — do NOT re-report unless the revision failed to address them:
 ${digest(seen2)}
 
-Enumerated ${opts.enumeratedItemsLabel} checklist — you and the other reviewer BOTH return this every round (it is NOT partitioned like the guideline rules): read EVERY artifact file's \`${opts.enumeratedItemsSection}\` section, enumerate each item in order as ${opts.enumeratedItemsLabel}1..${opts.enumeratedItemsLabel}n across the whole artifact, and return one ac_checklist entry per item ({ id: "${opts.enumeratedItemsLabel}<k>", verdict: pass|violation|na, evidence: one line }). An item the ${opts.artifactNoun} fails to satisfy, or that no step/test covers, is a BLOCKING finding carrying full guideline-rule weight — enumerating and checking these items is as load-bearing as the rule checklist. If no file has a \`${opts.enumeratedItemsSection}\` section, return \`ac_checklist: []\`.
+Enumerated ${opts.enumeratedItemsLabel} checklist — you and the other reviewer BOTH return this every round (it is NOT partitioned like the guideline rules): read EVERY artifact file's \`${opts.enumeratedItemsSection}\` section, enumerate each item in order as ${opts.enumeratedItemsLabel}1..${opts.enumeratedItemsLabel}n across the whole artifact, and return one ac_checklist entry per item ({ id: "${opts.enumeratedItemsLabel}<k>", verdict: pass|violation|na, evidence: one line }). An item the ${opts.artifactNoun} fails to satisfy, or that no step/test covers, is a BLOCKING finding — checking these is as load-bearing as the rule checklist. If no file has a \`${opts.enumeratedItemsSection}\` section, return \`ac_checklist: []\`.
 
 Severity: "blocking" = the plan as written produces wrong or missing work; "concern" = likely gap needing a fix or an explicit justification; "suggestion" = optional polish (never drives revision). Stable key format "<rule-id-or-gap>:<plan-location>". Confidence under ${cfg.confidenceMin} will be dropped.
 
@@ -1045,13 +1045,13 @@ ${item.validations.map((v) => `- ${v}`).join(`
 
 Commit your work on ${item.branch} with a Conventional-Commits message (\`<type>(${cfg.slug}): <description>\` — scope is the run slug, no \`${item.id}:\` title prefix; reference ${item.id} and the feedback fix in the body). If validations pass, commit and return status "implemented" with validations_green true. If you hit a blocker you cannot resolve (contradictory addendum, validation failure you cannot fix), commit what is safe, return status "blocked" with the blocker described — do NOT loop indefinitely.`;
   }
-  return `You are the implementation agent for deliverable ${item.id} of strapped run "${cfg.slug}". You have fresh context — everything you need is in the files below.
+  return `You are the implementation agent for deliverable ${item.id} of strapped run "${cfg.slug}" — fresh context; everything you need is below.
 
-Work EXCLUSIVELY inside the worktree: ${item.worktree} (branch ${item.branch}, based on ${item.base}). This deliverable targets repo "${item.repo}" — never touch ${item.repoRoot} directly.
+Work EXCLUSIVELY inside the worktree ${item.worktree} (branch ${item.branch}, based on ${item.base}). This deliverable targets repo "${item.repo}" — never touch ${item.repoRoot} directly.
 
 1. Read your deliverable plan in full: ${item.planFile}
 2. Read the shared research digest: ${cfg.dir}/research.md
-3. Read the project guidelines: every CLAUDE.md that applies (repo root at minimum).
+3. Read every applicable CLAUDE.md (repo root at minimum).
 ${item.resumeNote ? `
 This deliverable is being RESUMED. Prior state:
 ${item.resumeNote}
@@ -1062,7 +1062,7 @@ Before finishing, ALL validations must pass inside the worktree:
 ${item.validations.map((v) => `- ${v}`).join(`
 `)}
 
-Commit your work on ${item.branch} with a Conventional-Commits message (\`<type>(${cfg.slug}): <description>\` — scope is the run slug, no \`${item.id}:\` title prefix; reference ${item.id} in the body). If validations pass, commit and return status "implemented" with validations_green true. If you hit a blocker you cannot resolve (missing dependency, contradictory plan, validation failure you cannot fix), commit what is safe, return status "blocked" with the blocker described — do NOT loop indefinitely.`;
+Commit on ${item.branch} with a Conventional-Commits message (\`<type>(${cfg.slug}): <description>\` — scope is the run slug, no \`${item.id}:\` title prefix; reference ${item.id} in the body). If validations pass, return status "implemented" with validations_green true. If you hit a blocker you cannot resolve (missing dependency, contradictory plan, an unfixable validation failure), commit what is safe and return status "blocked" with the blocker described — do NOT loop indefinitely.`;
 }
 function fixPrompt(cfg, item, findings, round, recordSuffix) {
   return `You are the fix agent for deliverable ${item.id} of strapped run "${cfg.slug}", code-review round ${round}. Fresh context — everything you need is below.
@@ -1296,9 +1296,9 @@ Conventions you MUST follow for every file format: ${cfg.conventionsFile}
 Procedure:
 1. Read the source plan in full, then research each target repo's codebase thoroughly: architecture, the modules the ask touches, existing utilities to reuse, test patterns.
 2. Write ${cfg.dir}/research.md — a distilled digest (~300 lines max): architecture notes, key files with one-line roles, library/API findings, decisions with rationale, known pitfalls. This is the only research context implementers will ever see.
-3. Split the work into deliverables by discrete theme, forming a DAG: independent work has no deps, dependent work lists its parent deliverable ids. Keep one coherent theme in a single deliverable so a reviewer can grasp the whole change in one PR — split a theme into multiple deliverables only when its estimated meaningful diff (excluding generated code, dependency/lockfile bumps, generated clients/schemas, vendored code, and large fixtures) exceeds ~1,000 changed lines. Prefer a few cohesive, independently-shippable nodes over many fragments that scatter one theme across PRs. Assign each deliverable to exactly one target repo.
-4. Write one self-contained file per deliverable at ${cfg.dir}/deliverables/<id>-<kebab>.md per the conventions (frontmatter: id, title, deps, repo: <one of the target repo names above>, status: pending, branch: strapped/${cfg.slug}/<id>-<kebab>, base, worktree: null, pr: null, review_rounds_used: 0, feedback_rounds_used: 0, parked_reason: null, estimated_diff_lines; body: Context slice from your research, Files to touch, Implementation steps, Acceptance criteria, Tests, Out of scope). Set base per the cross-repo base rule: a deliverable's base is a parent branch WITHIN THE SAME repo, otherwise that repo's main (roots, and any cross-repo child, base on their own repo's main — you can never branch across repos). A fresh implementer seeded with ONLY this file plus research.md must be able to do the work.
-5. Cross-repo deps are ordering-only, NEVER a code dependency: a cross-repo child bases on its own repo's main and does not have its parent's unmerged code. Reject or restructure any plan where a cross-repo child has a true code dependency on its parent — either require the shared change to merge to the parent repo's main first, or keep both sides in the same repo/chain.
+3. Split the work into deliverables by discrete theme, forming a DAG: independent work has no deps, dependent work lists its parent deliverable ids. Meaningful diff = changed lines excluding generated code, dependency/lockfile bumps, generated clients/schemas, vendored code, and large fixtures. Keep one coherent theme in a single deliverable so a reviewer can grasp the whole change in one PR; split a theme only when its estimated meaningful diff exceeds ~1,000 lines. Prefer a few cohesive, independently-shippable nodes over many fragments that scatter one theme across PRs. Assign each deliverable to exactly one target repo.
+4. Write one self-contained file per deliverable at ${cfg.dir}/deliverables/<id>-<kebab>.md per the conventions (frontmatter: id, title, deps, repo: <one of the target repo names above>, status: pending, branch: strapped/${cfg.slug}/<id>-<kebab>, base, worktree: null, pr: null, review_rounds_used: 0, feedback_rounds_used: 0, parked_reason: null, estimated_diff_lines; body: Context slice from your research, Files to touch, Implementation steps, Acceptance criteria, Tests, Out of scope). A fresh implementer seeded with ONLY this file plus research.md must be able to do the work.
+5. Cross-repo base rule: a deliverable's base is a parent branch WITHIN THE SAME repo, otherwise that repo's main — roots and every cross-repo child base on their own repo's main; you can never branch across repos. Cross-repo deps are therefore ordering-only, never a code dependency: a cross-repo child does not have its parent's unmerged code. Reject or restructure any plan where a cross-repo child truly needs its parent's code — merge the shared change to the parent repo's main first, or keep both sides in the same repo/chain.
 6. Write ${cfg.dir}/manifest.md per the conventions (status: in-review, seed: ${cfg.seed}, budgets — record the EFFECTIVE budgets of this run: plan_rounds: ${cfg.planRounds}, code_rounds: ${cfg.codeRounds}, confidence_min: ${cfg.confidenceMin} — the repos: map listing every target repo above per the conventions — name, root, config path (repos: is an unordered set, no repo is special); the deliverables list with ids/files/repos/deps, theme summary, ASCII DAG sketch).
 7. After all plan artifacts are written, run \`node ${stateScript} commit ${cfg.dir}\` via Bash so the run's state root is git-backed from birth (it git-inits the state root if absent and commits the artifacts). Best-effort: proceed even if it reports an error.
 

src/eval/suites/harness/implementer.case.ts

@@ -31,7 +31,11 @@ const ITEM_REPO_ROOT = '/home/user/strapped'
 const ITEM_PLAN_FILE = `${FIXTURE_DIR}/deliverables/D1-dry-run-resolver.md`
 const ITEM_VALIDATIONS = ['bun run typecheck', 'bun run lint', 'bun test']
 
-const IMPLEMENTER_PROMPT = `You are the implementation agent for deliverable ${ITEM_ID} of strapped run "${FIXTURE_SLUG}". You have fresh context — everything you need is in the files below.
+// Baseline implementer prompt, verbatim from the pre-D4 non-addendum branch of
+// `implementPrompt`: filler lead-in ("You have fresh context — everything you
+// need is in the files below."), a verbose read-list item 3, and a doubled
+// "commit" in the closing paragraph.
+const IMPLEMENTER_BASELINE = `You are the implementation agent for deliverable ${ITEM_ID} of strapped run "${FIXTURE_SLUG}". You have fresh context — everything you need is in the files below.
 
 Work EXCLUSIVELY inside the worktree: ${ITEM_WORKTREE} (branch ${ITEM_BRANCH}, based on ${ITEM_BASE}). This deliverable targets repo "${ITEM_REPO}" — never touch ${ITEM_REPO_ROOT} directly.
 
@@ -46,11 +50,35 @@ ${ITEM_VALIDATIONS.map(v => `- ${v}`).join('\n')}
 
 Commit your work on ${ITEM_BRANCH} with a Conventional-Commits message (\`<type>(${FIXTURE_SLUG}): <description>\` — scope is the run slug, no \`${ITEM_ID}:\` title prefix; reference ${ITEM_ID} in the body). If validations pass, commit and return status "implemented" with validations_green true. If you hit a blocker you cannot resolve (missing dependency, contradictory plan, validation failure you cannot fix), commit what is safe, return status "blocked" with the blocker described — do NOT loop indefinitely.`
 
+// Candidate implementer prompt: filler lead-in folded into the header, read-list
+// item 3 tightened, and the doubled "commit" removed. Mirrors the compacted live
+// stage source.
+const IMPLEMENTER_CANDIDATE = `You are the implementation agent for deliverable ${ITEM_ID} of strapped run "${FIXTURE_SLUG}" — fresh context; everything you need is below.
+
+Work EXCLUSIVELY inside the worktree ${ITEM_WORKTREE} (branch ${ITEM_BRANCH}, based on ${ITEM_BASE}). This deliverable targets repo "${ITEM_REPO}" — never touch ${ITEM_REPO_ROOT} directly.
+
+1. Read your deliverable plan in full: ${ITEM_PLAN_FILE}
+2. Read the shared research digest: ${FIXTURE_DIR}/research.md
+3. Read every applicable CLAUDE.md (repo root at minimum).
+
+Implement exactly what the plan specifies — its acceptance criteria are the contract. Write the tests the plan names (integration-style, public interfaces). Stay in scope: anything under "Out of scope" is off limits; note side-discoveries in your summary instead of fixing them.
+
+Before finishing, ALL validations must pass inside the worktree:
+${ITEM_VALIDATIONS.map(v => `- ${v}`).join('\n')}
+
+Commit on ${ITEM_BRANCH} with a Conventional-Commits message (\`<type>(${FIXTURE_SLUG}): <description>\` — scope is the run slug, no \`${ITEM_ID}:\` title prefix; reference ${ITEM_ID} in the body). If validations pass, return status "implemented" with validations_green true. If you hit a blocker you cannot resolve (missing dependency, contradictory plan, an unfixable validation failure), commit what is safe and return status "blocked" with the blocker described — do NOT loop indefinitely.`
+
 export const implementerCase = defineCase({
   id: 'implementer',
   tags: ['implementer'],
   appendSystemPrompt: STRAPPED_CONTEXT,
-  prompt: IMPLEMENTER_PROMPT,
+  // `prompt` tracks the LIVE (compacted) stage source; `variants` records the
+  // baseline→candidate A/B this deliverable verified (`bun run eval --ab`).
+  prompt: IMPLEMENTER_CANDIDATE,
+  variants: {
+    baseline: { label: 'baseline', prompt: IMPLEMENTER_BASELINE },
+    candidate: { label: 'compacted', prompt: IMPLEMENTER_CANDIDATE },
+  },
   schema: asSchema(IMPLEMENT_SCHEMA),
   graders: [
     schemaConforms(),