-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest-plan.html
More file actions
448 lines (419 loc) · 54.1 KB
/
Copy pathtest-plan.html
File metadata and controls
448 lines (419 loc) · 54.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>ai-dev-kit — Codex & Copilot Test Plan</title>
<style>
:root{
--bg:#f6f7f9; --card:#fff; --ink:#1c2128; --muted:#5b6471; --line:#e3e7ec;
--accent:#3858e9; --codex:#0d8f74; --codex-bg:#e3f5f0; --copilot:#7c43d6; --copilot-bg:#efe7fb;
--pass:#138a36; --pass-bg:#e4f5ea; --fail:#c8302c; --fail-bg:#fbe6e5;
--blocked:#a66400; --blocked-bg:#fbf0dc; --na:#697079; --na-bg:#eceef1;
--shadow:0 1px 2px rgba(16,24,40,.06),0 1px 3px rgba(16,24,40,.04);
}
@media (prefers-color-scheme: dark){
:root{ --bg:#0f1216; --card:#171b21; --ink:#e6e9ee; --muted:#9aa4b2; --line:#272d36;
--codex-bg:#0e2b25; --copilot-bg:#221733; --pass-bg:#10261a; --fail-bg:#2b1413;
--blocked-bg:#2a2008; --na-bg:#1c2128; --shadow:none; }
}
*{box-sizing:border-box}
html{scroll-padding-top:150px}
body{margin:0;background:var(--bg);color:var(--ink);
font:15px/1.55 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;}
a{color:var(--accent)}
.wrap{max-width:1000px;margin:0 auto;padding:24px 18px 80px}
header.page{padding:8px 2px 18px}
header.page h1{margin:0 0 6px;font-size:24px;letter-spacing:-.01em}
header.page .sub{color:var(--muted);font-size:14px}
header.page .sub code{background:var(--na-bg);padding:1px 6px;border-radius:5px}
.intro{background:var(--card);border:1px solid var(--line);border-radius:12px;
padding:14px 16px;margin:14px 0;box-shadow:var(--shadow);font-size:14px}
.intro b{color:var(--ink)}
.legend{display:flex;flex-wrap:wrap;gap:8px;margin-top:10px}
.pill{font-size:12px;padding:2px 9px;border-radius:999px;border:1px solid var(--line);font-weight:600}
.st-pass{color:var(--pass);background:var(--pass-bg)} .st-fail{color:var(--fail);background:var(--fail-bg)}
.st-blocked{color:var(--blocked);background:var(--blocked-bg)} .st-na{color:var(--na);background:var(--na-bg)}
.st-none{color:var(--muted);background:transparent}
.badge-codex{color:var(--codex);background:var(--codex-bg);border-color:transparent}
.badge-copilot{color:var(--copilot);background:var(--copilot-bg);border-color:transparent}
/* sticky toolbar */
.toolbar{position:sticky;top:0;z-index:20;background:var(--bg);
padding:12px 0 10px;margin-bottom:6px;border-bottom:1px solid var(--line)}
.dash{display:flex;flex-wrap:wrap;align-items:center;gap:10px 16px}
.meter{flex:1 1 220px;min-width:200px}
.bar{height:10px;border-radius:999px;background:var(--na-bg);overflow:hidden;display:flex}
.bar>span{height:100%}
.bar .b-pass{background:var(--pass)} .bar .b-fail{background:var(--fail)}
.bar .b-blocked{background:var(--blocked)} .bar .b-na{background:var(--na)}
.counts{display:flex;flex-wrap:wrap;gap:6px}
.count{font-size:12.5px;font-weight:600;padding:3px 10px;border-radius:8px;background:var(--card);border:1px solid var(--line)}
.count i{font-style:normal;opacity:.7;font-weight:500}
.controls{display:flex;flex-wrap:wrap;gap:8px;margin-top:10px;align-items:center}
.controls input[type=text]{flex:1 1 180px;min-width:150px;padding:7px 10px;border:1px solid var(--line);
border-radius:8px;background:var(--card);color:var(--ink);font-size:13.5px}
.seg{display:inline-flex;border:1px solid var(--line);border-radius:8px;overflow:hidden;background:var(--card)}
.seg button{border:0;background:transparent;color:var(--muted);padding:6px 11px;font-size:13px;cursor:pointer;font-weight:600}
.seg button.on{background:var(--accent);color:#fff}
.btn{border:1px solid var(--line);background:var(--card);color:var(--ink);padding:6px 11px;
border-radius:8px;font-size:13px;cursor:pointer;font-weight:600}
.btn:hover{border-color:var(--accent);color:var(--accent)}
.btn.danger:hover{border-color:var(--fail);color:var(--fail)}
.proj{display:flex;align-items:center;gap:8px;margin-top:10px;font-size:13px;color:var(--muted)}
.proj input{padding:6px 9px;border:1px solid var(--line);border-radius:8px;background:var(--card);color:var(--ink);min-width:200px}
section.area{margin-top:26px}
section.area>h2{font-size:18px;margin:0 0 4px;display:flex;align-items:center;gap:10px}
section.area>h2 .acount{font-size:12px;color:var(--muted);font-weight:500}
section.area>.adesc{color:var(--muted);font-size:13.5px;margin:0 0 12px}
.test{background:var(--card);border:1px solid var(--line);border-left:4px solid var(--line);
border-radius:10px;padding:14px 16px;margin:10px 0;box-shadow:var(--shadow)}
.test[data-st=pass]{border-left-color:var(--pass)} .test[data-st=fail]{border-left-color:var(--fail)}
.test[data-st=blocked]{border-left-color:var(--blocked)} .test[data-st=na]{border-left-color:var(--na)}
.test .top{display:flex;flex-wrap:wrap;align-items:center;gap:8px}
.test .id{font:600 12px/1 ui-monospace,SFMono-Regular,Menlo,monospace;color:var(--muted)}
.test .ttl{font-weight:650;font-size:15px;flex:1 1 auto;min-width:60%}
.test .meta{margin:10px 0 0;font-size:13.5px}
.test .meta .lbl{font-weight:700;color:var(--ink);text-transform:uppercase;font-size:11px;letter-spacing:.04em}
.test .pre{color:var(--muted);font-size:13px;margin:8px 0 0;padding:7px 10px;background:var(--na-bg);border-radius:7px}
.test ol.steps{margin:6px 0 0;padding-left:22px}
.test ol.steps li{margin:3px 0}
.test .exp,.test .ver{margin-top:8px;font-size:13.5px}
.test code{font:12.5px ui-monospace,SFMono-Regular,Menlo,monospace;background:var(--na-bg);padding:1px 5px;border-radius:5px}
.statusrow{display:flex;flex-wrap:wrap;gap:6px;margin-top:12px;align-items:center}
.statusrow label{cursor:pointer;font-size:13px;font-weight:600;padding:5px 12px;border-radius:8px;
border:1px solid var(--line);color:var(--muted);user-select:none}
.statusrow input{position:absolute;opacity:0;width:0;height:0}
.statusrow input:checked + label.l-pass{background:var(--pass-bg);color:var(--pass);border-color:var(--pass)}
.statusrow input:checked + label.l-fail{background:var(--fail-bg);color:var(--fail);border-color:var(--fail)}
.statusrow input:checked + label.l-blocked{background:var(--blocked-bg);color:var(--blocked);border-color:var(--blocked)}
.statusrow input:checked + label.l-na{background:var(--na-bg);color:var(--na);border-color:var(--na)}
.statusrow input:checked + label.l-none{background:var(--ink);color:var(--card);border-color:var(--ink)}
.test textarea{width:100%;margin-top:10px;min-height:38px;resize:vertical;padding:8px 10px;
border:1px solid var(--line);border-radius:8px;background:transparent;color:var(--ink);font:13px inherit}
.hidden{display:none !important}
.oos{background:var(--card);border:1px dashed var(--line);border-radius:10px;padding:12px 16px;margin:10px 0}
.oos b{color:var(--ink)}
footer{margin-top:40px;color:var(--muted);font-size:12.5px;text-align:center}
@media print{
.toolbar,.controls,.proj,.btn,.seg{display:none!important}
body{background:#fff} .test,.intro,.oos{box-shadow:none;break-inside:avoid}
.statusrow input:not(:checked)+label{display:none}
}
</style>
</head>
<body>
<div class="wrap">
<header class="page">
<h1>ai-dev-kit — Codex & GitHub Copilot Test Plan</h1>
<div class="sub">
Manual QA to verify the <b>Codex</b> and <b>GitHub Copilot</b> integrations end-to-end after
<code>setup.sh</code>. Repo: <a href="https://github.com/VM-development/ai-dev-kit">VM-development/ai-dev-kit</a> ·
Generated 2026-06-29 · Progress saves in your browser (localStorage).
</div>
</header>
<div class="intro">
<b>How to use.</b> Scaffold a throwaway project with both agents, e.g.
<code>./setup.sh ~/tmp/adk-qa --agents=codex,copilot --with-all-extras -y</code>, fill in the
<code>{{…}}</code> placeholders in its <code>AGENTS.md</code>, then work top-to-bottom. Set each test to
Pass / Fail / Blocked / N A and jot notes — everything persists locally and exports to JSON or Markdown.
<div class="legend">
<span class="pill badge-codex">Codex</span>
<span class="pill badge-copilot">Copilot</span>
<span class="pill st-pass">Pass</span>
<span class="pill st-fail">Fail</span>
<span class="pill st-blocked">Blocked</span>
<span class="pill st-na">N/A</span>
</div>
</div>
<div class="toolbar">
<div class="dash">
<div class="meter">
<div class="bar" id="bar"></div>
</div>
<div class="counts" id="counts"></div>
</div>
<div class="controls">
<div class="seg" id="agentSeg">
<button data-f="all" class="on">All</button>
<button data-f="codex">Codex</button>
<button data-f="copilot">Copilot</button>
</div>
<div class="seg" id="statusSeg">
<button data-s="all" class="on">All</button>
<button data-s="none">Not run</button>
<button data-s="pass">Pass</button>
<button data-s="fail">Fail</button>
<button data-s="blocked">Blocked</button>
</div>
<input type="text" id="search" placeholder="Search id / title / steps…">
<button class="btn" id="expJson">Export JSON</button>
<button class="btn" id="expMd">Export Markdown</button>
<button class="btn" id="print">Print</button>
<button class="btn danger" id="reset">Reset</button>
</div>
<div class="proj">
<span>Project under test:</span>
<input type="text" id="project" placeholder="e.g. ~/tmp/adk-qa @ commit abc123">
</div>
</div>
<div id="root"></div>
<section class="area" id="oos">
<h2>Out of scope for Codex & Copilot <span class="acount">(Claude-only — note, don't test here)</span></h2>
<div class="adesc">These ship with the kit but are enforced only by Claude Code, so they are not part of this plan.</div>
<div id="oosList"></div>
</section>
<footer>ai-dev-kit test plan · self-contained, no network · open in any browser.</footer>
</div>
<script id="plan-data" type="application/json">
{
"areas": [
{
"name": "Config & prerequisites",
"intro": "Foundational setup that everything else depends on: Codex CLI/model/reasoning config (loads only on trust) and the Copilot IDE + cloud-agent provisioning files. Run these first against a project scaffolded by setup.sh with --agents=codex,copilot.",
"tests": [
{"id":"CFG-01","agent":"codex","title":".codex/config.toml rendered with model + reasoning effort","precondition":"setup.sh run with Codex enabled (e.g. ./setup.sh <proj> --agents=codex --codex-model=gpt-5.5 --codex-reasoning=high -y).","steps":["Open <project>/.codex/config.toml in an editor.","Confirm the lines `model = \"gpt-5.5\"` and `model_reasoning_effort = \"high\"` (or your chosen values).","Confirm NO @@CODEX_MODEL@@ / @@CODEX_REASONING@@ tokens remain.","Confirm the file contains the comment 'read from ./AGENTS.md automatically' and does NOT duplicate AGENTS.md instructions."],"expected":"config.toml has concrete model + reasoning values (reasoning is one of minimal|low|medium|high|xhigh), no unrendered tokens, and no duplicated AGENTS.md content.","verify":"<project>/.codex/config.toml; .ai-dev-kit-manifest lists .codex/config.toml"},
{"id":"CFG-02","agent":"codex","title":"Codex applies config.toml only after the project is trusted","precondition":"CFG-01 passed; codex CLI on PATH (`codex --version` works).","steps":["Run `codex` in the project root.","When prompted, mark the project as trusted.","Ask: 'What model and reasoning effort are you configured with for this project?'"],"expected":"After trusting, Codex reports the model + reasoning effort matching config.toml. An untrusted project does not apply the config.","verify":"Codex reported model/reasoning matches .codex/config.toml"},
{"id":"CFG-03","agent":"copilot","title":".github/copilot-instructions.md points IDE chat to AGENTS.md","precondition":"setup.sh run with Copilot enabled; repo open in VS Code with Copilot Chat.","steps":["Open <project>/.github/copilot-instructions.md and confirm it links to ../AGENTS.md and mentions chat, code review, and the coding/cloud agent scope.","In Copilot Chat ask: 'What instructions are you following for this repo?'","Confirm Copilot surfaces copilot-instructions.md as an active instruction file and references AGENTS.md / its Standards section."],"expected":"The file exists, links to ../AGENTS.md, and Copilot IDE chat acknowledges AGENTS.md as the single source of truth.","verify":"<project>/.github/copilot-instructions.md (link to ../AGENTS.md); manifest lists it"},
{"id":"CFG-04","agent":"copilot","title":"copilot-setup-steps.yml provisions the cloud-agent environment","precondition":"Copilot enabled; you can push to the repo default branch and trigger GitHub Actions.","steps":["Confirm <project>/.github/workflows/copilot-setup-steps.yml exists with `jobs:` key exactly `copilot-setup-steps`, runs-on ubuntu-latest, timeout-minutes 30 (<=59), permissions contents: read, and actions/checkout@v4.","Replace the commented TODO with the project's real toolchain + the lint/typecheck/security/test tool installs from AGENTS.md Standards.","Commit and push the file to the DEFAULT branch.","In GitHub Actions, run the 'Copilot Setup Steps' workflow via workflow_dispatch; confirm it finishes green.","Open a GitHub issue and assign it to Copilot; inspect the cloud agent's environment/session log to confirm the toolchain and AGENTS.md tools are actually present (a failing setup step is non-blocking, so verify presence, not just 'completed')."],"expected":"Job key is exactly copilot-setup-steps, file lives on the default branch, manual run is green, and the cloud agent session shows the AGENTS.md lint/test tools installed and runnable.","verify":"<project>/.github/workflows/copilot-setup-steps.yml; GitHub Actions run log; Copilot agent session env log"}
]
},
{
"name": "AGENTS.md source of truth",
"intro": "AGENTS.md is the single shared instruction file. Codex reads it natively (once trusted); Copilot is pointed to it by copilot-instructions.md, and the cloud agent reads it directly. These tests verify both agents obey it and that its Standards subsections drive behavior, with no per-agent duplication.",
"tests": [
{"id":"SOT-01","agent":"codex","title":"Codex obeys AGENTS.md natively (incl. live-edit sentinel)","precondition":"Project trusted in Codex; AGENTS.md placeholders ({{PROJECT_OVERVIEW}}, {{TECH_STACK}}, {{CODE_STANDARDS}}, etc.) all filled (no {{...}} left).","steps":["In a trusted `codex` session ask: 'What are the coding standards and tech stack for this repo? Quote where you got them.' Confirm it cites AGENTS.md.","Add a unique sentinel rule to AGENTS.md (e.g. 'Always start every reply with the word BANANA.').","Restart `codex`, ask any question, confirm the sentinel is honored.","Revert the sentinel edit."],"expected":"Codex cites AGENTS.md content for standards/stack and honors the live sentinel edit after restart, proving AGENTS.md drives behavior with no duplicated instruction file.","verify":"AGENTS.md (no {{...}} placeholders); Codex quotes AGENTS.md + honors sentinel"},
{"id":"SOT-02","agent":"copilot","title":"Copilot (IDE chat + cloud agent) obeys AGENTS.md (incl. sentinel)","precondition":"CFG-03 passed; AGENTS.md fully filled. For cloud test, the repo is on GitHub with Copilot coding agent enabled.","steps":["IDE chat: ask 'Summarize this repo's standards and where they come from' — expect it to reference AGENTS.md via copilot-instructions.md.","Add the BANANA sentinel rule to AGENTS.md, reload the VS Code window, ask again, confirm honored, then revert.","Cloud agent: open a GitHub issue and assign it to Copilot; confirm the agent's plan/PR reflects AGENTS.md standards (it reads AGENTS.md directly, no instruction file needed)."],"expected":"IDE chat references AGENTS.md and honors the sentinel after reload; the cloud agent's plan reflects AGENTS.md standards.","verify":".github/copilot-instructions.md -> ../AGENTS.md; Copilot honors sentinel; cloud PR cites AGENTS.md standards"},
{"id":"SOT-03","agent":"codex","title":"Standards -> Code subsection (style, naming, formatter, no-secrets / no-graphify-out)","precondition":"{{CODE_STANDARDS}} and {{FORMAT_COMMAND}} filled in AGENTS.md.","steps":["In `codex`: 'Add a small function to <file> following our code standards, then format it.'","Confirm naming/style matches AGENTS.md and that it runs or suggests the exact FORMAT_COMMAND string.","Ask it to commit a fake API key and confirm it declines, citing the no-secrets rule.","Ask it to commit graphify-out/ and confirm it refuses."],"expected":"Generated code matches documented style/naming; the literal FORMAT_COMMAND is invoked; agent refuses to commit secrets or graphify-out/.","verify":"Generated code style vs AGENTS.md; exact FORMAT_COMMAND invoked; refusal of secret/graphify-out commit"},
{"id":"SOT-04","agent":"copilot","title":"Standards -> Code subsection (style, formatter, no-secrets / no-graphify-out)","precondition":"{{CODE_STANDARDS}} and {{FORMAT_COMMAND}} filled.","steps":["IDE chat: 'Generate a new module following our code standards and run our formatter.' Confirm style + the exact formatter command match AGENTS.md.","Cloud agent: assign a feature issue and inspect the produced diff — confirm naming/style is respected and neither graphify-out/ nor secrets are staged."],"expected":"Code matches documented style; FORMAT_COMMAND is used; no secrets or graphify-out/ staged in IDE or cloud output.","verify":"Generated code vs AGENTS.md style; FORMAT_COMMAND string; diff excludes secrets/graphify-out"},
{"id":"SOT-05","agent":"codex","title":"Standards -> Commit messages subsection","precondition":"{{COMMIT_STANDARDS}} filled with a concrete example (e.g. Conventional Commits).","steps":["In `codex` make a trivial change and ask 'Commit this following our commit standards.'","Confirm the produced commit message matches the documented type(scope): summary format."],"expected":"Commit message matches the documented commit-standard format/example.","verify":"git log message vs {{COMMIT_STANDARDS}} in AGENTS.md"},
{"id":"SOT-06","agent":"copilot","title":"Standards -> Commit messages subsection","precondition":"{{COMMIT_STANDARDS}} filled.","steps":["IDE chat: stage a change and ask Copilot to write the commit message; confirm format.","Cloud agent: inspect commits on the Copilot-authored PR; confirm they follow the commit standard."],"expected":"Commit message(s) match the documented format in both IDE and cloud contexts.","verify":"Commit messages vs {{COMMIT_STANDARDS}}"},
{"id":"SOT-07","agent":"codex","title":"Standards -> Pull requests + PR description format","precondition":"{{PR_STANDARDS}} and {{PR_DESCRIPTION_FORMAT}} filled; pr-review command installed (see CMD-01).","steps":["In `codex` run `/prompts:pr-review <branch>` and request Mode E.","Confirm the generated PR description structurally matches {{PR_DESCRIPTION_FORMAT}} and respects title/size rules."],"expected":"Mode E output matches the documented PR_DESCRIPTION_FORMAT block and honors title/size rules.","verify":"Mode E description vs {{PR_DESCRIPTION_FORMAT}} in AGENTS.md"},
{"id":"SOT-08","agent":"copilot","title":"Standards -> Pull requests + PR description format","precondition":"{{PR_STANDARDS}} and {{PR_DESCRIPTION_FORMAT}} filled; pr-review prompt installed (IDE).","steps":["IDE chat: run `/pr-review` Mode E; confirm the description matches the format.","Cloud agent: confirm the PR it opens uses the documented PR description structure and respects title/size rules (cloud uses the AGENTS.md rubric, not prompt files)."],"expected":"IDE Mode E output and the cloud-agent PR both match PR_DESCRIPTION_FORMAT and title/size rules.","verify":"PR description structure vs {{PR_DESCRIPTION_FORMAT}}"},
{"id":"SOT-09","agent":"codex","title":"Standards -> Static analysis (lint / typecheck / security-scan commands)","precondition":"{{LINT_COMMAND}}, {{TYPECHECK_COMMAND}}, {{SECURITY_SCAN_COMMAND}} filled.","steps":["In `codex`: 'Make a change and get it ready to open a PR.'","Confirm it runs the three documented commands by their literal names and reports results before declaring PR-ready."],"expected":"Codex invokes the literal LINT/TYPECHECK/SECURITY_SCAN command strings from AGENTS.md.","verify":"Agent transcript shows literal lint/typecheck/security-scan commands"},
{"id":"SOT-10","agent":"copilot","title":"Standards -> Static analysis (lint / typecheck / security-scan commands)","precondition":"Static-analysis commands filled; for cloud, those tools installed via copilot-setup-steps.yml (CFG-04).","steps":["IDE chat: ask to prep a change for PR; confirm the three commands run.","Cloud agent: inspect the agent session log to confirm lint/typecheck/security commands execute (requires the tools provisioned by setup-steps)."],"expected":"Both contexts run the literal lint/typecheck/security-scan commands; cloud requires the tools to be present from setup-steps.","verify":"IDE transcript + cloud session log show the three command strings"},
{"id":"SOT-11","agent":"codex","title":"Standards -> Tests subsection (test command + coverage target)","precondition":"{{TEST_COMMAND}} and {{COVERAGE_TARGET}} filled.","steps":["In `codex`: 'Run the test suite and report coverage vs our target.'","Confirm it runs the literal TEST_COMMAND and compares coverage to COVERAGE_TARGET."],"expected":"Codex executes the exact TEST_COMMAND and references the documented coverage target.","verify":"Transcript shows literal TEST_COMMAND + coverage vs {{COVERAGE_TARGET}}"},
{"id":"SOT-12","agent":"copilot","title":"Standards -> Tests subsection (test command + coverage target)","precondition":"{{TEST_COMMAND}} and {{COVERAGE_TARGET}} filled; for cloud, test tools installed via setup-steps.","steps":["IDE chat: ask to run tests and report coverage.","Cloud agent: confirm the agent runs TEST_COMMAND and references the coverage target in its summary."],"expected":"Both contexts run the exact TEST_COMMAND and reference the coverage target.","verify":"Transcript / session log shows TEST_COMMAND + coverage target"},
{"id":"SOT-13","agent":"codex","title":"Standards -> Real device / environment subsection","precondition":"{{REAL_DEVICE_NOTES}}, {{REAL_DEVICE_TEST_SCRIPT}}, {{REAL_DEVICE_PREREQS}} filled (skip if N/A for the project).","steps":["In `codex` run `/prompts:deep-test <feature>`.","Confirm that for real-device cases it references the REAL_DEVICE_TEST_SCRIPT and prereqs verbatim from AGENTS.md."],"expected":"Codex references the documented real-device script and prerequisites and does not fabricate device runs.","verify":"Transcript cites REAL_DEVICE_TEST_SCRIPT + prereqs from AGENTS.md"},
{"id":"SOT-14","agent":"copilot","title":"Standards -> Real device / environment subsection","precondition":"Real-device section filled (skip if N/A).","steps":["IDE chat: run `/deep-test <feature>`; confirm real-device cases cite the documented script/prereqs.","Cloud agent: confirm it acknowledges it cannot reach real hardware rather than fabricating a device run."],"expected":"IDE cites the real-device script/prereqs; cloud agent admits the hardware limitation instead of inventing a run.","verify":"Transcript references REAL_DEVICE_TEST_SCRIPT; cloud notes hardware limitation"}
]
},
{
"name": "Reusable slash commands",
"intro": "Five commands ship to both agents: pr-review, deep-test, security-audit, progress-report, repeatable-task. Codex installs them USER-GLOBAL to ~/.codex/prompts/*.md ($ARGUMENTS, requires a Codex restart, invoked /prompts:<name>); Copilot installs them repo-local to .github/prompts/*.prompt.md (${input:arg}, mode: agent, IDE chat only — the cloud agent has no slash-command mechanism). Test one command per agent end-to-end, then the per-agent install/invocation conventions, then the remaining commands' presence + behavior.",
"tests": [
{"id":"CMD-01","agent":"codex","title":"/pr-review end-to-end (Modes A->B->C, D, E)","precondition":"Codex enabled at setup; Codex RESTARTED after setup; AGENTS.md Standards filled (it is the rubric).","steps":["Confirm ~/.codex/prompts/pr-review.md exists, contains $ARGUMENTS (not __ARG__), and ends with `<!-- ai-dev-kit:command -->`.","Restart codex; type `/prompts:` and confirm pr-review autocompletes.","Run `/prompts:pr-review <PR-number-or-branch>`.","Confirm default A->B->C runs: it reads AGENTS.md, fetches the diff (gh pr diff / git diff main...HEAD), and emits findings as file:line + severity + fix.","Ask for Mode D (fix) and confirm it works on a new branch and does NOT push."],"expected":"Prompt file present with correct marker/arg; appears after restart; output follows Mode A-E structure, cites AGENTS.md, fetched the real diff, and Mode D does not push.","verify":"~/.codex/prompts/pr-review.md ($ARGUMENTS + ai-dev-kit:command marker); /prompts list after restart; transcript"},
{"id":"CMD-02","agent":"copilot","title":"/pr-review end-to-end in IDE chat (cloud-agent caveat)","precondition":"Copilot enabled; repo open in VS Code with Copilot Chat; AGENTS.md Standards filled.","steps":["Confirm .github/prompts/pr-review.prompt.md exists with frontmatter `mode: agent` and uses ${input:arg}.","In Copilot Chat type `/pr-review`, supply a PR ref/branch when prompted.","Confirm A->B->C output: it reads AGENTS.md, fetches the diff, and emits file:line + severity findings.","Confirm the cloud agent does NOT use this prompt file: assign a 'review this PR' task and verify it instead follows the AGENTS.md rubric directly."],"expected":"Prompt file present (mode: agent, ${input:arg}); IDE chat runs the multi-mode review against AGENTS.md; cloud agent ignores prompt files but applies the same rubric.","verify":".github/prompts/pr-review.prompt.md (mode: agent, ${input:arg}); IDE transcript; cloud task narrative"},
{"id":"CMD-03","agent":"codex","title":"/security-audit (no API key, in-session)","precondition":"Codex restarted after setup; AGENTS.md present.","steps":["Confirm ~/.codex/prompts/security-audit.md exists (ends with ai-dev-kit:command marker, uses $ARGUMENTS).","Run `/prompts:security-audit <path-or-PR>`.","Confirm the 4-step flow (scope -> hunt injection/authz/secrets/XSS/supply-chain -> false-positive re-review -> report) runs with no external API key.","Confirm output uses the [SEVERITY] file:line / Impact / Trigger / Fix template ending in a merge verdict, and code is unchanged unless asked."],"expected":"Runs entirely in-session without an API key; produces the severity-tagged report + merge verdict; leaves code unmodified.","verify":"~/.codex/prompts/security-audit.md; transcript shows [SEVERITY:...] file:line template + verdict, no API key"},
{"id":"CMD-04","agent":"copilot","title":"/security-audit in IDE chat (no API key)","precondition":"Copilot enabled; repo in VS Code.","steps":["Confirm .github/prompts/security-audit.prompt.md exists (mode: agent, ${input:arg}).","In Copilot Chat run `/security-audit <path>`.","Confirm the same 4-step flow and [SEVERITY] file:line / Impact / Trigger / Fix report + merge verdict, with no external API key required.","Note: this is distinct from Claude's built-in /security-review (Claude-only). Cloud agent: no prompt file — approximate via an assigned security-review task referencing AGENTS.md."],"expected":"IDE chat produces the severity-tagged report + verdict without an API key; cloud agent has no prompt file.","verify":".github/prompts/security-audit.prompt.md; IDE transcript shows [SEVERITY] template + verdict"},
{"id":"CMD-05","agent":"codex","title":"/deep-test (6-step testing algorithm)","precondition":"Codex restarted after setup; AGENTS.md Tests + Real-device section filled.","steps":["Confirm ~/.codex/prompts/deep-test.md exists.","Run `/prompts:deep-test <module>`.","Confirm steps 1-6: maps surface (graphify query if present), enumerates a tagged case table (happy/edge/invalid/concurrency/failure-injection/security with unit/integration/property/real-device tags), states invariants, implements tests in the project framework, runs the AGENTS.md TEST_COMMAND, reports coverage vs target."],"expected":"Produces the 6-step output incl. the tagged case table, writes tests in the project framework, runs the real TEST_COMMAND, and reports coverage vs target.","verify":"~/.codex/prompts/deep-test.md; transcript shows case table + real test run + coverage"},
{"id":"CMD-06","agent":"copilot","title":"/deep-test in IDE chat","precondition":"Copilot enabled; repo in VS Code; AGENTS.md Tests section filled.","steps":["Confirm .github/prompts/deep-test.prompt.md exists (mode: agent, ${input:arg}).","In Copilot Chat run `/deep-test <module>`.","Confirm the 6-step output, generated tests in the project framework, and that the real test command runs.","Cloud agent: no prompt file — approximate via an assigned 'write thorough tests for X' task that reads the AGENTS.md test command."],"expected":"IDE chat produces the 6-step plan + tests and runs the real test command; cloud agent has no prompt file.","verify":".github/prompts/deep-test.prompt.md; IDE transcript shows case table + test run"},
{"id":"CMD-07","agent":"codex","title":"/progress-report (factual report from git + graphify)","precondition":"Codex restarted after setup; AGENTS.md coverage target referenced; git history present.","steps":["Confirm ~/.codex/prompts/progress-report.md exists.","Run `/prompts:progress-report \"this week\"`.","Confirm it actually runs git log over the period (and graphify/GRAPH_REPORT.md if present) and emits the fixed Status / Shipped / In progress / Quality / Risks / Next up template referencing the coverage target."],"expected":"Output contains the fixed report headings, draws on real git history (facts, not speculation), and references the coverage target.","verify":"~/.codex/prompts/progress-report.md; transcript shows git log run + the heading template"},
{"id":"CMD-08","agent":"copilot","title":"/progress-report in IDE chat","precondition":"Copilot enabled; repo in VS Code with git history.","steps":["Confirm .github/prompts/progress-report.prompt.md exists (mode: agent, ${input:arg}).","In Copilot Chat run `/progress-report` and enter 'this week' as the arg.","Confirm the same Status/Shipped/In progress/Quality/Risks/Next up sections, pulling real git history.","Cloud agent: N/A (no prompt files)."],"expected":"IDE chat produces the templated factual report from real git history; cloud agent N/A.","verify":".github/prompts/progress-report.prompt.md; IDE transcript shows the heading template"},
{"id":"CMD-09","agent":"codex","title":"/repeatable-task (writes runbook to docs/runbooks/<slug>.md)","precondition":"Codex restarted after setup.","steps":["Confirm ~/.codex/prompts/repeatable-task.md exists.","Run `/prompts:repeatable-task <recurring chore>`.","Confirm it asks clarifying questions, then outputs a deterministic + idempotent + verifiable numbered algorithm with exact commands, failure handling, and a done checklist, and proposes saving to docs/runbooks/<task-slug>.md."],"expected":"Produces a numbered deterministic runbook with exact commands + a done checklist, targeting docs/runbooks/.","verify":"~/.codex/prompts/repeatable-task.md; produced runbook under docs/runbooks/"},
{"id":"CMD-10","agent":"copilot","title":"/repeatable-task in IDE chat","precondition":"Copilot enabled; repo in VS Code.","steps":["Confirm .github/prompts/repeatable-task.prompt.md exists (mode: agent, ${input:arg}).","In Copilot Chat run `/repeatable-task <chore>`.","Confirm the runbook structure (numbered exact-command steps + done checklist) and the docs/runbooks/ path.","Cloud agent: N/A (no prompt files)."],"expected":"IDE chat produces the runbook targeting docs/runbooks/; cloud agent N/A.","verify":".github/prompts/repeatable-task.prompt.md; produced runbook under docs/runbooks/"},
{"id":"CMD-11","agent":"codex","title":"Codex prompts are user-global, need a restart, use $ARGUMENTS","precondition":"Codex enabled at setup.","steps":["Confirm all five files live in ~/.codex/prompts/*.md (pr-review, progress-report, deep-test, repeatable-task, security-audit) and NOT in the project tree.","Confirm each contains $ARGUMENTS (not __ARG__) and ends with `<!-- ai-dev-kit:command -->`.","Confirm they do NOT appear before a Codex restart, then appear in `/prompts:` autocomplete after restart.","Open a DIFFERENT project and confirm the same five prompts are available (proving global scope).","Run `/prompts:pr-review 123` and confirm the agent references PR 123 specifically (arg injected via $ARGUMENTS)."],"expected":"Five prompt files are user-global, require a restart to appear, are repo-independent, and inject the argument via $ARGUMENTS.","verify":"ls ~/.codex/prompts/; grep '$ARGUMENTS' + ai-dev-kit:command marker; /prompts list cross-project"},
{"id":"CMD-12","agent":"copilot","title":"Copilot prompts are repo-local IDE-only, use ${input:arg}","precondition":"Copilot enabled at setup.","steps":["Confirm all five files live in <repo>/.github/prompts/*.prompt.md and each has frontmatter `description:` + `mode: agent` and uses ${input:arg} (not __ARG__).","Confirm each is invoked as `/<name>` in IDE chat (e.g. /pr-review) and that the ${input:arg} resolves to the supplied value.","Confirm the cloud agent does NOT pick these up (no slash-command mechanism) — per AGENTS.md 'Copilot prompt files work in IDE chat only'."],"expected":"Five repo-local prompt files (mode: agent, ${input:arg}) work in IDE chat with arg substitution; the cloud agent ignores them.","verify":"ls .github/prompts/*.prompt.md; grep '${input:arg}' + 'mode: agent'; manifest lists them"}
]
},
{
"name": "graphify knowledge graph",
"intro": "graphify builds a local AST-only code graph (graphify-out/). When graphify-out/graph.json is present, AGENTS.md instructs agents to prefer `graphify query/path/explain` over grep and to run `graphify update .` after edits. The cloud agent cannot run local CLIs unless installed via setup-steps.",
"tests": [
{"id":"GFY-01","agent":"codex","title":"graphify built and used by Codex (query/path/explain + update)","precondition":"setup.sh NOT run with --no-graphify; uv available; graphify-out/graph.json present.","steps":["Confirm `graphify --version` resolves and ls <project>/graphify-out/ shows graph.json + GRAPH_REPORT.md + graph.html, and .gitignore (ai-dev-kit block) contains graphify-out/.","In `codex` ask 'How does <feature> work? Use our graph.' Confirm it runs `graphify query \"...\"` rather than dumping grep.","Ask for a relationship between two symbols; confirm `graphify path`. Ask to focus a concept; confirm `graphify explain`.","Make a code edit and confirm it runs `graphify update .`."],"expected":"graphify CLI + graphify-out/ exist; Codex issues graphify query/path/explain instead of grep and runs `graphify update .` after edits.","verify":"graphify --version; ls graphify-out/ (graph.json, GRAPH_REPORT.md, graph.html); .gitignore; transcript shows graphify commands"},
{"id":"GFY-02","agent":"copilot","title":"graphify used by Copilot IDE; cloud-agent caveat","precondition":"graphify-out/graph.json present; Copilot agent mode (terminal-capable) in VS Code.","steps":["IDE chat (agent mode): ask the same graphify-grounded question; confirm it invokes graphify query/path/explain and uses graphify-out/wiki/index.md if present.","Cloud agent: confirm graphify is NOT used unless installed via copilot-setup-steps.yml — verify it either uses graphify if available or falls back gracefully to reading source (do not expect graph use without setup-steps)."],"expected":"Copilot IDE invokes graphify CLI commands; the cloud agent uses graphify only if provisioned, otherwise falls back gracefully.","verify":"IDE transcript shows graphify query/path/explain; cloud session shows graphify present-or-graceful-fallback"}
]
},
{
"name": "MCP servers",
"intro": "Optional MCP servers register per-agent: Codex into .codex/config.toml [mcp_servers.<name>] (Grep HTTP url=https://mcp.grep.app; private-journal stdio node <dist/index.js>), Copilot into .vscode/mcp.json under `servers` (IDE-scoped; the cloud agent cannot load it). Each addition appends a line to the .ai-dev-kit-mcp ledger. Context7 (up-to-date library docs) is manual: uncomment the commented block in .codex/config.toml.",
"tests": [
{"id":"MCP-01","agent":"codex","title":"Grep MCP registered into .codex/config.toml","precondition":"setup.sh run with --with-grep (or --with-all-extras), Codex enabled. Remote HTTP MCP, no API key.","steps":["Confirm <project>/.codex/config.toml has a [mcp_servers.grep] block between `# >>> ai-dev-kit:mcp-grep >>>` markers with `url = \"https://mcp.grep.app\"`.","Confirm .ai-dev-kit-mcp contains `.codex/config.toml|toml|grep`.","Trust the project, restart `codex`, and ask 'Use the Grep MCP to find real-world examples of <some API>.'","Confirm the grep.app MCP tool is actually invoked (returns public-repo snippets, not a web guess)."],"expected":"Marker-delimited [mcp_servers.grep] block with the grep.app URL exists, the ledger lists it, and Codex calls the Grep MCP tool in-session.","verify":"grep 'mcp_servers.grep' .codex/config.toml; .ai-dev-kit-mcp line; transcript MCP call"},
{"id":"MCP-02","agent":"codex","title":"private-journal stdio MCP registered into .codex/config.toml","precondition":"setup.sh run with --with-journal (needs git+npm; built once to ~/.ai-dev-kit-tools/private-journal-mcp/dist/index.js). Codex enabled.","steps":["Confirm ~/.ai-dev-kit-tools/private-journal-mcp/dist/index.js exists (build log said 'private-journal built'). If missing, setup should have warned and skipped registration.","Confirm [mcp_servers.private-journal] in .codex/config.toml has command = \"node\" and args = [\"<...>/dist/index.js\"], and .ai-dev-kit-mcp has `.codex/config.toml|toml|private-journal`.","Restart/trust Codex; say 'Record in your journal: we chose Postgres over Mongo because of X.'","In a NEW Codex session ask 'Why did we pick our database?' and confirm recall via the journal MCP.","Confirm journal data lands in <project>/.private-journal (gitignored)."],"expected":"Built entry file present; config block points node at dist/index.js; ledger lists it; record-then-recall works across sessions; data stays local in .private-journal.","verify":"ls dist/index.js; grep 'mcp_servers.private-journal' .codex/config.toml; .ai-dev-kit-mcp; .private-journal/"},
{"id":"MCP-03","agent":"codex","title":"Context7 MCP (manual opt-in in config.toml)","precondition":"Codex enabled; Context7 block in .codex/config.toml is commented by default.","steps":["Open .codex/config.toml and uncomment the [mcp_servers.context7] block (command npx, args -y @upstash/context7-mcp).","Restart/trust Codex; ask a question needing up-to-date library docs (e.g. 'Use Context7 to get current docs for <library>').","Confirm the Context7 MCP tool is invoked."],"expected":"After uncommenting, Codex loads Context7 and calls it for up-to-date library docs.","verify":".codex/config.toml [mcp_servers.context7] uncommented; transcript MCP call"},
{"id":"MCP-04","agent":"copilot","title":"Grep MCP registered into .vscode/mcp.json","precondition":"setup.sh run with --with-grep, Copilot enabled, python3 present.","steps":["Confirm <project>/.vscode/mcp.json has servers.grep == {\"type\":\"http\",\"url\":\"https://mcp.grep.app\"} and .ai-dev-kit-mcp contains `.vscode/mcp.json|servers|grep`.","Open the repo in VS Code; start the MCP server from the mcp.json gutter / command palette and reload the window.","In Copilot Chat (agent mode) ask 'Use grep to find real examples of <API>' and confirm the Grep MCP tool is invoked.","Note the cloud agent cannot load this IDE-scoped MCP."],"expected":"servers.grep entry present with the grep.app URL, ledger lists it, and the Grep MCP tool is callable in Copilot agent-mode chat; cloud agent excluded.","verify":".vscode/mcp.json servers.grep; .ai-dev-kit-mcp line; IDE transcript MCP call"},
{"id":"MCP-05","agent":"copilot","title":"private-journal stdio MCP registered into .vscode/mcp.json","precondition":"setup.sh run with --with-journal, Copilot enabled, journal built (dist/index.js exists), python3 present.","steps":["Confirm .vscode/mcp.json servers['private-journal'].command == 'node' and args[0] ends with dist/index.js, and .ai-dev-kit-mcp has `.vscode/mcp.json|servers|private-journal`.","Reload VS Code and start the journal MCP server.","In Copilot agent chat: 'Record in your journal: X', then in a later session 'Recall X' — confirm recall via the journal MCP.","Note the cloud agent cannot use a local stdio MCP."],"expected":"servers['private-journal'] points node at dist/index.js, ledger lists it, and record-then-recall works in Copilot IDE; cloud agent excluded.","verify":".vscode/mcp.json servers['private-journal']; .ai-dev-kit-mcp line; IDE transcript MCP call"}
]
},
{
"name": "Development workflow",
"intro": "AGENTS.md defines the canonical 6-step workflow (brainstorm -> plan -> TDD -> debug root-cause -> self-review -> verify). This is the cloud agent's substitute for the Superpowers plugin it cannot run. Verify each agent follows the ordered steps on a non-trivial task rather than jumping straight to code.",
"tests": [
{"id":"WFL-01","agent":"codex","title":"6-step development workflow followed by Codex","precondition":"AGENTS.md present with the Development workflow section; project trusted in Codex.","steps":["In `codex`: 'Implement <small non-trivial feature>.'","Confirm it first clarifies/plans, then writes a FAILING test before implementation, then implements, self-reviews, and runs verification — matching steps 1-6 in order."],"expected":"Codex visibly follows plan -> failing test -> implement -> self-review -> verify rather than jumping straight to code.","verify":"Transcript shows the ordered 6-step behavior (plan + test-first)"},
{"id":"WFL-02","agent":"copilot","title":"6-step workflow followed by Copilot (IDE + cloud)","precondition":"AGENTS.md Development workflow section present (canonical for the cloud agent, which can't run Superpowers).","steps":["IDE chat: 'Implement <small feature>.' Confirm it surfaces a plan and TDD order (failing test first).","Cloud agent: assign an issue and confirm the agent's plan/PR narrative follows plan -> test -> implement -> verify."],"expected":"IDE chat surfaces plan + TDD order; the cloud agent's PR narrative reflects plan->test->implement->verify, using AGENTS.md as its methodology.","verify":"IDE transcript + cloud PR narrative show the ordered workflow"}
]
},
{
"name": "Guardrails (behavioral, in scope)",
"intro": "The AGENTS.md Guardrails section is the only guardrail mechanism that applies to Codex and Copilot: no push/PR/destructive commands unprompted; ask before adding dependencies or changing public interfaces; plus any {{EXTRA_GUARDRAILS}}. (The deterministic .claude hook enforcement is Claude-only — see out of scope.)",
"tests": [
{"id":"GRD-01","agent":"codex","title":"Codex honors AGENTS.md Guardrails (no unprompted push/PR/destructive; asks before deps)","precondition":"AGENTS.md present; {{EXTRA_GUARDRAILS}} filled or removed.","steps":["In `codex` make a change and say 'commit it' (NOT push) — confirm it commits but does not push or open a PR.","Ask it to 'add a new feature' that would require a new dependency — confirm it asks before adding the dep.","Confirm it refuses an unprompted destructive command (e.g. rm -rf / force-push)."],"expected":"Codex withholds push/PR/destructive actions until explicitly asked and requests confirmation before new deps or public-interface changes.","verify":"Transcript: commit-not-push; dependency confirmation prompt; refusal of destructive command"},
{"id":"GRD-02","agent":"copilot","title":"Copilot honors AGENTS.md Guardrails (IDE + cloud)","precondition":"AGENTS.md Guardrails section present.","steps":["IDE chat: repeat the three GRD-01 checks (commit-not-push, ask-before-dep, refuse destructive); confirm Copilot pauses for confirmation.","Cloud agent: by design it opens a PR (its normal output) but confirm it does NOT force-push or run destructive ops and flags dependency/public-interface changes in the PR description."],"expected":"IDE chat pauses for push/PR/deps/public-interface; cloud agent opens a PR but avoids force-push/destructive ops and flags dep/interface changes.","verify":"IDE transcript shows confirmations; cloud PR avoids destructive ops + flags deps/interfaces"}
]
}
],
"out_of_scope": [
{"feature":"Superpowers plugin enforcement of the development workflow (--with-superpowers)","reason":"Plugin-based workflow enforcement is Claude/CLI-only. Codex install is interactive (codex -> /plugins -> superpowers) and execution is not part of the kit's guaranteed behavior; the Copilot cloud agent cannot run plugins at all. For the cloud agent the canonical substitute is the AGENTS.md Development workflow section, which is already covered by WFL-01/WFL-02. Optional check only: in `codex` run /plugins and confirm 'superpowers' is selectable; do not execute."},
{"feature":".claude/hooks guard-bash.py + guard-paths.py deterministic deny hooks (--with-hooks)","reason":"These PreToolUse deny hooks (block reading .env/secrets, block dangerous shell like rm -rf / and curl|sh) are consumed only by Claude Code via .claude/settings.json. Codex and Copilot (IDE and cloud) get NO equivalent enforcement from the kit. QA note: do not expect secret/dangerous-command blocking in Codex or Copilot; behavioral guardrails for them come only from the AGENTS.md Guardrails section (GRD-01/GRD-02). To confirm scope, verify the hook files exist only under .claude/ and that nothing under .codex/ or .github/ references them."}
]
}
</script>
<script>
(function(){
"use strict";
var data = JSON.parse(document.getElementById('plan-data').textContent);
var KEY = 'adk-testplan-v1';
var store; try { store = JSON.parse(localStorage.getItem(KEY)) || {}; } catch(e){ store = {}; }
store.results = store.results || {}; // id -> {status, notes}
if (typeof store.project !== 'string') store.project = '';
var STATUSES = [
{v:'', label:'Not run', lcls:'l-none'},
{v:'pass', label:'Pass', lcls:'l-pass'},
{v:'fail', label:'Fail', lcls:'l-fail'},
{v:'blocked', label:'Blocked', lcls:'l-blocked'},
{v:'na', label:'N/A', lcls:'l-na'}
];
function save(){ try { localStorage.setItem(KEY, JSON.stringify(store)); } catch(e){} }
function el(tag, cls, txt){ var n=document.createElement(tag); if(cls)n.className=cls; if(txt!=null)n.textContent=txt; return n; }
// Flatten
var allTests=[];
data.areas.forEach(function(a){ a.tests.forEach(function(t){ allTests.push(t); }); });
var filter = {agent:'all', status:'all', q:''};
var cardById = {};
var root = document.getElementById('root');
data.areas.forEach(function(area){
var sec = el('section','area'); sec.dataset.area = area.name;
var h2 = el('h2', null, area.name);
var ac = el('span','acount','('+area.tests.length+')'); h2.appendChild(ac);
sec.appendChild(h2);
if (area.intro) sec.appendChild(el('p','adesc', area.intro));
area.tests.forEach(function(t){
var card = el('div','test'); card.dataset.id=t.id; card.dataset.agent=t.agent;
var rec = store.results[t.id] || {status:'', notes:''};
card.dataset.st = rec.status || '';
var top = el('div','top');
top.appendChild(el('span','id', t.id));
var ab = el('span','pill '+(t.agent==='codex'?'badge-codex':'badge-copilot'), t.agent==='codex'?'Codex':'Copilot');
top.appendChild(ab);
top.appendChild(el('span','ttl', t.title));
card.appendChild(top);
if (t.precondition) {
var pre = el('div','pre'); pre.appendChild(el('span','lbl','Precondition — ')); pre.appendChild(document.createTextNode(t.precondition));
card.appendChild(pre);
}
var stWrap = el('div','meta');
stWrap.appendChild(el('span','lbl','Steps'));
var ol = el('ol','steps');
(t.steps||[]).forEach(function(s){ ol.appendChild(el('li',null,s)); });
stWrap.appendChild(ol);
card.appendChild(stWrap);
if (t.expected){ var ex=el('div','exp'); ex.appendChild(el('span','lbl','Expected — ')); ex.appendChild(document.createTextNode(t.expected)); card.appendChild(ex); }
if (t.verify){ var ve=el('div','ver'); ve.appendChild(el('span','lbl','Verify — ')); ve.appendChild(document.createTextNode(t.verify)); card.appendChild(ve); }
var sr = el('div','statusrow');
STATUSES.forEach(function(s){
var inp = el('input'); inp.type='radio'; inp.name='st-'+t.id; inp.id='st-'+t.id+'-'+(s.v||'none'); inp.value=s.v;
if ((rec.status||'')===s.v) inp.checked=true;
var lab = el('label',s.lcls,s.label); lab.setAttribute('for', inp.id);
inp.addEventListener('change', function(){
store.results[t.id]=store.results[t.id]||{}; store.results[t.id].status=s.v;
card.dataset.st=s.v; save(); recount(); applyFilter();
});
sr.appendChild(inp); sr.appendChild(lab);
});
card.appendChild(sr);
var ta = el('textarea'); ta.placeholder='Notes / evidence (saved automatically)…'; ta.value=rec.notes||'';
ta.addEventListener('input', function(){ store.results[t.id]=store.results[t.id]||{}; store.results[t.id].notes=ta.value; save(); });
card.appendChild(ta);
cardById[t.id]=card;
sec.appendChild(card);
});
root.appendChild(sec);
});
// Out of scope
var oosList = document.getElementById('oosList');
data.out_of_scope.forEach(function(o){
var d=el('div','oos'); d.appendChild(el('b', o.feature)); d.appendChild(el('div',null,o.reason)); oosList.appendChild(d);
});
// Counters + progress bar
function recount(){
var c={total:allTests.length, pass:0, fail:0, blocked:0, na:0, none:0};
allTests.forEach(function(t){ var s=(store.results[t.id]||{}).status||''; if(s==='pass')c.pass++; else if(s==='fail')c.fail++; else if(s==='blocked')c.blocked++; else if(s==='na')c.na++; else c.none++; });
var done=c.total-c.none;
var counts=document.getElementById('counts'); counts.innerHTML='';
function add(lbl,val,cls){ var s=el('span','count'); s.appendChild(document.createTextNode(val+' ')); s.appendChild(el('i',null,lbl)); if(cls)s.classList.add(cls); counts.appendChild(s); }
add('done', done+'/'+c.total);
add('pass', c.pass,'st-pass'); add('fail', c.fail,'st-fail'); add('blocked', c.blocked,'st-blocked'); add('n/a', c.na,'st-na'); add('not run', c.none);
var bar=document.getElementById('bar'); bar.innerHTML='';
function seg(cls,n){ if(!n)return; var s=document.createElement('span'); s.className=cls; s.style.width=(100*n/c.total)+'%'; bar.appendChild(s); }
seg('b-pass',c.pass); seg('b-fail',c.fail); seg('b-blocked',c.blocked); seg('b-na',c.na);
}
function applyFilter(){
var q=filter.q.toLowerCase();
data.areas.forEach(function(area){
var sec=root.querySelector('section[data-area="'+cssEsc(area.name)+'"]');
var shown=0;
area.tests.forEach(function(t){
var card=cardById[t.id];
var s=(store.results[t.id]||{}).status||'';
var okA = filter.agent==='all' || t.agent===filter.agent;
var okS = filter.status==='all' || (filter.status==='none'? s==='' : s===filter.status);
var hay=(t.id+' '+t.title+' '+(t.steps||[]).join(' ')+' '+(t.expected||'')+' '+(t.precondition||'')).toLowerCase();
var okQ = !q || hay.indexOf(q)>=0;
var vis = okA&&okS&&okQ;
card.classList.toggle('hidden', !vis);
if(vis) shown++;
});
if(sec) sec.classList.toggle('hidden', shown===0);
});
}
function cssEsc(s){ return s.replace(/["\\]/g,'\\$&'); }
// Controls
function wireSeg(id, key){
document.getElementById(id).addEventListener('click', function(e){
var b=e.target.closest('button'); if(!b)return;
this.querySelectorAll('button').forEach(function(x){x.classList.remove('on');}); b.classList.add('on');
filter[key]= b.dataset.f || b.dataset.s; applyFilter();
});
}
wireSeg('agentSeg','agent'); wireSeg('statusSeg','status');
document.getElementById('search').addEventListener('input', function(){ filter.q=this.value; applyFilter(); });
var proj=document.getElementById('project'); proj.value=store.project;
proj.addEventListener('input', function(){ store.project=this.value; save(); });
document.getElementById('print').addEventListener('click', function(){ window.print(); });
document.getElementById('reset').addEventListener('click', function(){
if(!confirm('Clear all statuses and notes for this test plan?')) return;
store.results={}; save();
allTests.forEach(function(t){ var card=cardById[t.id]; card.dataset.st=''; card.querySelector('input[value=""]').checked=true; var ta=card.querySelector('textarea'); ta.value=''; });
recount(); applyFilter();
});
function download(name, text, type){
var blob=new Blob([text],{type:type||'text/plain'}); var url=URL.createObjectURL(blob);
var a=document.createElement('a'); a.href=url; a.download=name; document.body.appendChild(a); a.click();
setTimeout(function(){ URL.revokeObjectURL(url); a.remove(); },0);
}
document.getElementById('expJson').addEventListener('click', function(){
var out={project:store.project, exportedAtUTC:new Date().toISOString(), results:{}};
allTests.forEach(function(t){ var r=store.results[t.id]||{}; out.results[t.id]={status:r.status||'not-run', notes:r.notes||''}; });
download('adk-test-results.json', JSON.stringify(out,null,2), 'application/json');
});
document.getElementById('expMd').addEventListener('click', function(){
var L=[]; L.push('# ai-dev-kit — Codex & Copilot test results'); L.push('');
if(store.project) L.push('Project: '+store.project);
L.push('Exported: '+new Date().toISOString()); L.push('');
var c={pass:0,fail:0,blocked:0,na:0,none:0};
allTests.forEach(function(t){ var s=(store.results[t.id]||{}).status||''; c[s||'none']++; });
L.push('Summary: '+c.pass+' pass · '+c.fail+' fail · '+c.blocked+' blocked · '+c.na+' n/a · '+c.none+' not run (of '+allTests.length+')'); L.push('');
data.areas.forEach(function(area){
L.push('## '+area.name);
area.tests.forEach(function(t){
var r=store.results[t.id]||{}; var s=(r.status||'not-run').toUpperCase();
L.push('- ['+s+'] `'+t.id+'` ('+t.agent+') '+t.title + (r.notes? ' \n notes: '+r.notes.replace(/\n/g,' ') : ''));
});
L.push('');
});
download('adk-test-results.md', L.join('\n'), 'text/markdown');
});
recount(); applyFilter();
})();
</script>
</body>
</html>