From 08929319e9017e2abb21071b219a47ed6fbb426f Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:27:21 -0400 Subject: [PATCH 01/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20template?= =?UTF-8?q?:=20move=20the=20verdict=20of=20readiness.md=20to=20its=20own?= =?UTF-8?q?=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readiness.go nears the size limit. The verdict check moves to readiness_verdict.go, with no change of behavior. Co-Authored-By: Claude Opus 5.5 --- sdk/extensionrepo/readiness.go | 76 ----------------------- sdk/extensionrepo/readiness_verdict.go | 83 ++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 76 deletions(-) create mode 100644 sdk/extensionrepo/readiness_verdict.go diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index 4d1eb4dc..39a300e5 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -1,9 +1,7 @@ package extensionrepo import ( - "errors" "fmt" - "os" "regexp" "slices" "strings" @@ -116,77 +114,3 @@ func readinessFindings(readiness string) ([]finding, []string) { } return findings, problems } - -// readinessProblems reports a readiness.md without a reviewer or a passing -// verdict, or whose commit is not the code of the working tree. The verdict -// must follow from the findings and from the limits of spec.md (#409). Only -// readiness.md may differ from that commit, so the review can be committed -// after it. -func readinessProblems(limits []limit) []string { - const fix = "Fix: ask an agent that did not write the code to run the readiness review of the nicely-extension skill, then start readiness.md with the lines Commit: , Reviewer: , and Verdict: " - const verdictFix = "Fix: write the verdict that the findings and the limits give, as the readiness review of the nicely-extension skill says" - data, err := os.ReadFile("readiness.md") - if errors.Is(err, os.ErrNotExist) { - return []string{"readiness.md: no readiness verdict is recorded. " + fix} - } else if err != nil { - return []string{"readiness.md: " + err.Error()} - } - // readiness.md starts with the lines Commit:, Reviewer:, and Verdict:, - // as the skill says. So finish never reads them in fenced code (#413). - lines := strings.SplitN(string(data), "\n", 4) - header := strings.Join(lines[:min(3, len(lines))], "\n") - commit, verdict := readinessCommit.FindStringSubmatch(header), readinessVerdict.FindStringSubmatch(header) - if commit == nil || verdict == nil { - return []string{"readiness.md: it holds no line Commit: or Verdict: . " + fix} - } - findings, problems := readinessFindings(string(data)) - if !readinessReviewer.MatchString(header) { - problems = append(problems, "readiness.md: it names no reviewer. "+fix) - } - switch v := verdict[1]; v { - case "Ready": - for _, f := range findings { - if f.severity == "block" || f.severity == "warn" { - problems = append(problems, fmt.Sprintf("readiness.md:%d: the verdict is Ready, but this finding has the severity %s. %s", f.line, f.severity, verdictFix)) - } - } - for _, l := range limits { - problems = append(problems, fmt.Sprintf("spec.md:%d: the verdict is Ready, but spec.md states the limit `%s`. %s", l.line, l.id, verdictFix)) - } - case "Ready with stated limits": - for _, f := range findings { - switch { - case f.severity == "block": - problems = append(problems, fmt.Sprintf("readiness.md:%d: the verdict is Ready with stated limits, but this finding has the severity block. %s", f.line, verdictFix)) - case f.severity == "warn" && !slices.ContainsFunc(limits, func(l limit) bool { return l.id == f.limit && l.accepted }): - problems = append(problems, fmt.Sprintf("readiness.md:%d: this warn names no limit that the user accepted in spec.md. Fix: name the spec ID of its accepted limit in a line - **Limit**: ``, or write the verdict Needs work", f.line)) - } - } - case "Needs work": - problems = append(problems, "readiness.md: the verdict is Needs work. Fix: start the next round from its findings") - default: - problems = append(problems, fmt.Sprintf("readiness.md: the verdict %q is not Ready, Ready with stated limits, or Needs work. %s", v, fix)) - } - sha := commit[1] - if _, err := updateOutput("git", "rev-parse", "--verify", "--quiet", sha+"^{commit}"); err != nil { - return append(problems, fmt.Sprintf("readiness.md: Git knows no commit %s. %s", sha, fix)) - } - changed, err := updateOutput("git", "diff", "--name-only", sha, "--") - if err != nil { - return append(problems, "readiness.md: "+err.Error()) - } - added, err := updateOutput("git", "ls-files", "--others", "--exclude-standard") - if err != nil { - return append(problems, "readiness.md: "+err.Error()) - } - var files []string - for _, file := range strings.Split(string(changed)+string(added), "\n") { - if file != "" && file != "readiness.md" && !slices.Contains(files, file) { - files = append(files, file) - } - } - if len(files) != 0 { - problems = append(problems, fmt.Sprintf("readiness.md: the review is of %s, and these files changed since: %s. Fix: run the readiness review again on this commit", sha, strings.Join(files, ", "))) - } - return problems -} diff --git a/sdk/extensionrepo/readiness_verdict.go b/sdk/extensionrepo/readiness_verdict.go new file mode 100644 index 00000000..946a121f --- /dev/null +++ b/sdk/extensionrepo/readiness_verdict.go @@ -0,0 +1,83 @@ +package extensionrepo + +import ( + "errors" + "fmt" + "os" + "slices" + "strings" +) + +// readinessProblems reports a readiness.md without a reviewer or a passing +// verdict, or whose commit is not the code of the working tree. The verdict +// must follow from the findings and from the limits of spec.md (#409). Only +// readiness.md may differ from that commit, so the review can be committed +// after it. +func readinessProblems(limits []limit) []string { + const fix = "Fix: ask an agent that did not write the code to run the readiness review of the nicely-extension skill, then start readiness.md with the lines Commit: , Reviewer: , and Verdict: " + const verdictFix = "Fix: write the verdict that the findings and the limits give, as the readiness review of the nicely-extension skill says" + data, err := os.ReadFile("readiness.md") + if errors.Is(err, os.ErrNotExist) { + return []string{"readiness.md: no readiness verdict is recorded. " + fix} + } else if err != nil { + return []string{"readiness.md: " + err.Error()} + } + // readiness.md starts with the lines Commit:, Reviewer:, and Verdict:, + // as the skill says. So finish never reads them in fenced code (#413). + lines := strings.SplitN(string(data), "\n", 4) + header := strings.Join(lines[:min(3, len(lines))], "\n") + commit, verdict := readinessCommit.FindStringSubmatch(header), readinessVerdict.FindStringSubmatch(header) + if commit == nil || verdict == nil { + return []string{"readiness.md: it holds no line Commit: or Verdict: . " + fix} + } + findings, problems := readinessFindings(string(data)) + if !readinessReviewer.MatchString(header) { + problems = append(problems, "readiness.md: it names no reviewer. "+fix) + } + switch v := verdict[1]; v { + case "Ready": + for _, f := range findings { + if f.severity == "block" || f.severity == "warn" { + problems = append(problems, fmt.Sprintf("readiness.md:%d: the verdict is Ready, but this finding has the severity %s. %s", f.line, f.severity, verdictFix)) + } + } + for _, l := range limits { + problems = append(problems, fmt.Sprintf("spec.md:%d: the verdict is Ready, but spec.md states the limit `%s`. %s", l.line, l.id, verdictFix)) + } + case "Ready with stated limits": + for _, f := range findings { + switch { + case f.severity == "block": + problems = append(problems, fmt.Sprintf("readiness.md:%d: the verdict is Ready with stated limits, but this finding has the severity block. %s", f.line, verdictFix)) + case f.severity == "warn" && !slices.ContainsFunc(limits, func(l limit) bool { return l.id == f.limit && l.accepted }): + problems = append(problems, fmt.Sprintf("readiness.md:%d: this warn names no limit that the user accepted in spec.md. Fix: name the spec ID of its accepted limit in a line - **Limit**: ``, or write the verdict Needs work", f.line)) + } + } + case "Needs work": + problems = append(problems, "readiness.md: the verdict is Needs work. Fix: start the next round from its findings") + default: + problems = append(problems, fmt.Sprintf("readiness.md: the verdict %q is not Ready, Ready with stated limits, or Needs work. %s", v, fix)) + } + sha := commit[1] + if _, err := updateOutput("git", "rev-parse", "--verify", "--quiet", sha+"^{commit}"); err != nil { + return append(problems, fmt.Sprintf("readiness.md: Git knows no commit %s. %s", sha, fix)) + } + changed, err := updateOutput("git", "diff", "--name-only", sha, "--") + if err != nil { + return append(problems, "readiness.md: "+err.Error()) + } + added, err := updateOutput("git", "ls-files", "--others", "--exclude-standard") + if err != nil { + return append(problems, "readiness.md: "+err.Error()) + } + var files []string + for _, file := range strings.Split(string(changed)+string(added), "\n") { + if file != "" && file != "readiness.md" && !slices.Contains(files, file) { + files = append(files, file) + } + } + if len(files) != 0 { + problems = append(problems, fmt.Sprintf("readiness.md: the review is of %s, and these files changed since: %s. Fix: run the readiness review again on this commit", sha, strings.Join(files, ", "))) + } + return problems +} From dcdfd122fdaf468feaa19944e86378dd07dceb11 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:36:31 -0400 Subject: [PATCH 02/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20template?= =?UTF-8?q?:=20move=20the=20receipt=20refusals=20of=20the=20release=20proo?= =?UTF-8?q?f=20to=20its=20helpers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit template-release-test.sh nears the size limit. Its loop over the unsuccessful receipts moves to template-release-helpers.sh, with no change of behavior. Co-Authored-By: Claude Opus 5.5 --- scripts/template-release-helpers.sh | 16 ++++++++++++++++ scripts/template-release-test.sh | 11 +---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/scripts/template-release-helpers.sh b/scripts/template-release-helpers.sh index c237f384..8ccf6140 100644 --- a/scripts/template-release-helpers.sh +++ b/scripts/template-release-helpers.sh @@ -17,3 +17,19 @@ fct_release_receipt() { mkdir -p "${scratch}/release-cache/nicely/signoff/test-${sha}" PATH="/usr/bin:${PATH}" jq -n --arg sha "${sha}" '{sha:$sha,os:"test",tools:{git:"test"},checks:[{command:["just","check"],exit:0}],signoff:{command:["gh","signoff","--commit",$sha],exit:0}}' >"${scratch}/release-cache/nicely/signoff/test-${sha}/receipt.json" } + +# fct_release_refused_receipts requires that each unsuccessful form of the +# receipt $3 of the repository $1 refuses the release, then restores it. +fct_release_refused_receipts() { + local repo="${1}" scratch="${2}" receipt="${3}" previous status + cp "${receipt}" "${scratch}/valid-signoff.json" + for previous in '.checks[0].exit = 1' '.signoff.exit = 1' '.sha = "previous"' 'del(.checks[0].exit)' '.signoff.exit = null'; do + PATH="/usr/bin:${PATH}" jq "${previous}" "${scratch}/valid-signoff.json" >"${receipt}" + status=0 + fct_release_attempt "${repo}" "${scratch}" v0.1.0 || status=$? + if [[ "${status}" == 0 ]] || ! grep -qF 'just signoff' "${scratch}/release.out"; then + fct_fail 'release accepts an unsuccessful receipt' + fi + done + cp "${scratch}/valid-signoff.json" "${receipt}" +} diff --git a/scripts/template-release-test.sh b/scripts/template-release-test.sh index e5ffe01f..7a514dbc 100644 --- a/scripts/template-release-test.sh +++ b/scripts/template-release-test.sh @@ -37,16 +37,7 @@ SCRIPT fct_release_receipt "${repo}" "${scratch}" sha="$(git -C "${repo}" rev-parse HEAD)" receipt="${scratch}/release-cache/nicely/signoff/test-${sha}/receipt.json" - cp "${receipt}" "${scratch}/valid-signoff.json" - for previous in '.checks[0].exit = 1' '.signoff.exit = 1' '.sha = "previous"' 'del(.checks[0].exit)' '.signoff.exit = null'; do - PATH="/usr/bin:${PATH}" jq "${previous}" "${scratch}/valid-signoff.json" >"${receipt}" - status=0 - fct_release_attempt "${repo}" "${scratch}" v0.1.0 || status=$? - if [[ "${status}" == 0 ]] || ! grep -qF 'just signoff' "${scratch}/release.out"; then - fct_fail 'release accepts an unsuccessful receipt' - fi - done - cp "${scratch}/valid-signoff.json" "${receipt}" + fct_release_refused_receipts "${repo}" "${scratch}" "${receipt}" printf '\nDirty change\n' >>"${repo}/README.md" status=0 From e71f0f9bfc8618247d58918f5c6bbb671ad2532c Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:31:58 -0400 Subject: [PATCH 03/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20release?= =?UTF-8?q?=20needs=20just=20finish=20in=20the=20signoff=20receipt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A receipt with only just check, from before #415 or written by hand, let just release tag a repository whose loop was not done. Release now needs a just finish check with exit 0 in the receipt. Part of #416 and #417. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/extensions.md | 1 + scripts/template-release-helpers.sh | 9 ++++++++- sdk/extensionrepo/release_receipt.go | 11 ++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/north-star/contract/extensions.md b/docs/north-star/contract/extensions.md index 164c8f9b..5844bba9 100644 --- a/docs/north-star/contract/extensions.md +++ b/docs/north-star/contract/extensions.md @@ -23,6 +23,7 @@ An author outside Nicely keeps an extension in a Git repository of its own, whic An author releases a version with `just release vX.Y.Z [fixture]`. The SDK requires a clean HEAD equal to the fetched default branch of `origin`. A successful signoff receipt must name that SHA, with exit 0 for every check and signoff. +Its checks must hold `just finish`, so a receipt with only `just check` never releases. Release runs `just acceptance [fixture]`, then creates and pushes an annotated tag whose message holds that signoff receipt. It executes the fenced `sh` block under `## Install` in `README.md`, with `TAG` replaced by the new tag. The install uses scratch HOME and XDG folders, without the caller's keys or installed extensions. diff --git a/scripts/template-release-helpers.sh b/scripts/template-release-helpers.sh index 8ccf6140..25457e6a 100644 --- a/scripts/template-release-helpers.sh +++ b/scripts/template-release-helpers.sh @@ -15,7 +15,7 @@ fct_release_receipt() { local repo="${1}" scratch="${2}" sha sha="$(git -C "${repo}" rev-parse HEAD)" mkdir -p "${scratch}/release-cache/nicely/signoff/test-${sha}" - PATH="/usr/bin:${PATH}" jq -n --arg sha "${sha}" '{sha:$sha,os:"test",tools:{git:"test"},checks:[{command:["just","check"],exit:0}],signoff:{command:["gh","signoff","--commit",$sha],exit:0}}' >"${scratch}/release-cache/nicely/signoff/test-${sha}/receipt.json" + PATH="/usr/bin:${PATH}" jq -n --arg sha "${sha}" '{sha:$sha,os:"test",tools:{git:"test"},checks:[{command:["just","finish"],exit:0},{command:["just","check"],exit:0}],signoff:{command:["gh","signoff","--commit",$sha],exit:0}}' >"${scratch}/release-cache/nicely/signoff/test-${sha}/receipt.json" } # fct_release_refused_receipts requires that each unsuccessful form of the @@ -31,5 +31,12 @@ fct_release_refused_receipts() { fct_fail 'release accepts an unsuccessful receipt' fi done + # A receipt without just finish, as before #415 or written by hand, cannot authorize a release (#416). + PATH="/usr/bin:${PATH}" jq '.checks |= map(select(.command[-1] != "finish"))' "${scratch}/valid-signoff.json" >"${receipt}" + status=0 + fct_release_attempt "${repo}" "${scratch}" v0.1.0 || status=$? + if [[ "${status}" == 0 ]] || ! grep -qF 'holds no successful just finish' "${scratch}/release.out"; then + fct_fail 'release accepts a receipt without just finish' + fi cp "${scratch}/valid-signoff.json" "${receipt}" } diff --git a/sdk/extensionrepo/release_receipt.go b/sdk/extensionrepo/release_receipt.go index 1f37b248..db3528e8 100644 --- a/sdk/extensionrepo/release_receipt.go +++ b/sdk/extensionrepo/release_receipt.go @@ -10,6 +10,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "strings" "time" @@ -49,6 +50,7 @@ func releaseSigned(sha string) (signoffReceipt, error) { if err != nil { return signoffReceipt{}, err } + unfinished := false for _, path := range paths { data, err := os.ReadFile(path) if err != nil { @@ -72,9 +74,16 @@ func releaseSigned(sha string) (signoffReceipt, error) { for _, check := range r.Checks { valid = valid && check.Exit == 0 && len(check.Command) != 0 } - if valid { + // A receipt of an extension repository holds just finish too, so a + // receipt with only just check never releases (#416). + finished := slices.ContainsFunc(r.Checks, func(c signoffCheck) bool { return len(c.Command) != 0 && c.Command[len(c.Command)-1] == "finish" }) + if valid && finished { return r, nil } + unfinished = unfinished || valid + } + if unfinished { + return signoffReceipt{}, fmt.Errorf("the receipt for %s holds no successful just finish; run just signoff for this pushed SHA, which runs just finish first", sha) } return signoffReceipt{}, fmt.Errorf("no successful receipt for %s; run just signoff for this pushed SHA", sha) } From 61927861ca27f54310d1dccbe2aaaa605ad1fb54 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:32:24 -0400 Subject: [PATCH 04/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20contract?= =?UTF-8?q?:=20move=20the=20checks=20of=20just=20finish=20to=20their=20own?= =?UTF-8?q?=20part?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit extension-repository.md nears the size limit. The list of what just finish checks moves to finish-gate.md, with no change of rule. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract.md | 1 + .../north-star/contract/extension-repository.md | 12 +----------- docs/north-star/contract/finish-gate.md | 17 +++++++++++++++++ template/AGENTS.md.jinja | 2 +- 4 files changed, 20 insertions(+), 12 deletions(-) create mode 100644 docs/north-star/contract/finish-gate.md diff --git a/docs/north-star/contract.md b/docs/north-star/contract.md index 8653dabe..af7d461d 100644 --- a/docs/north-star/contract.md +++ b/docs/north-star/contract.md @@ -48,6 +48,7 @@ Each part is one file, in reading order. - [Spec IDs](contract/spec-ids.md) - [Generic cases](contract/generic-cases.md) - [Extension repository](contract/extension-repository.md) + - [Finish gate](contract/finish-gate.md) - [Update an extension](contract/update-an-extension.md) - [Operations](contract/operations.md) - [Records and evidence](contract/records-and-evidence.md) diff --git a/docs/north-star/contract/extension-repository.md b/docs/north-star/contract/extension-repository.md index 47d5c5f7..7d18b67f 100644 --- a/docs/north-star/contract/extension-repository.md +++ b/docs/north-star/contract/extension-repository.md @@ -36,17 +36,7 @@ Public repositories also run the same steps on `macos-15`, and both runners are The repository tests itself through the [test kit](test-kit.md). Its `justfile` has nine recipes: - `just check` runs every check: `go mod tidy -diff`, `gofmt`, `go vet`, and `go test`, whose test kit installs the extension into scratch folders -- `just finish` is the exit of the loop of the skill, and `just check` stays green for an incomplete round. `just finish` fails on each of these: - - A missing `vision.md`, `spec.md`, or `qa.md` - - A line of `vision.md`, `spec.md`, or `qa.md` that starts with `Fill:` after its indent, also in fenced code - - A line under `## Open lines` - - A line under **Limits.** that is no table row with a spec ID, the state `limit`, and the user's acceptance - - A `readiness.md` whose first three lines hold no `Commit: `, no `Reviewer: `, or no verdict `Ready` or `Ready with stated limits` - - A reviewed commit that differs from the working tree outside `readiness.md` - - A finding of `readiness.md` without its own `###` heading under `## Findings`, or without one severity: `block`, `warn`, or `advise` - - A severity under `## Findings` outside the line `- **Severity**:` of a finding, such as in a table, in fenced code, or in prose - - `Ready` with a `block` or `warn` finding, or with a limit in `spec.md` - - `Ready with stated limits` with a `block` finding, or with a `warn` whose **Limit** names no accepted limit +- `just finish` is the exit of the loop of the skill, and `just check` stays green for an incomplete round. [Finish gate](finish-gate.md) lists what it checks - `just signoff` runs `just finish` and `just check` on the committed SHA in isolation, then signs that SHA, as [Local checks](../decisions/D016-local-ci.md) defines - `just release [fixture]` publishes the signed SHA and proves the README install, as [Release and install](extensions.md#release-and-install) defines - `just install` builds the executable and exports its sidecars from that build. It replaces the previous install of the slug in the extensions folder of [Discovery](discovery.md), the executable last diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md new file mode 100644 index 00000000..5b429936 --- /dev/null +++ b/docs/north-star/contract/finish-gate.md @@ -0,0 +1,17 @@ +# Finish gate + +This part says what `just finish` checks in an extension repository, as [Extension repository](extension-repository.md) defines it. +`just finish` is the exit of the loop of the `nicely-extension` skill. `just check` stays green for an incomplete round. + +`just finish` fails on each of these: + +- A missing `vision.md`, `spec.md`, or `qa.md` +- A line of `vision.md`, `spec.md`, or `qa.md` that starts with `Fill:` after its indent, also in fenced code +- A line under `## Open lines` +- A line under **Limits.** that is no table row with a spec ID, the state `limit`, and the user's acceptance +- A `readiness.md` whose first three lines hold no `Commit: `, no `Reviewer: `, or no verdict `Ready` or `Ready with stated limits` +- A reviewed commit that differs from the working tree outside `readiness.md` +- A finding of `readiness.md` without its own `###` heading under `## Findings`, or without one severity: `block`, `warn`, or `advise` +- A severity under `## Findings` outside the line `- **Severity**:` of a finding, such as in a table, in fenced code, or in prose +- `Ready` with a `block` or `warn` finding, or with a limit in `spec.md` +- `Ready with stated limits` with a `block` finding, or with a `warn` whose **Limit** names no accepted limit diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index 4c76ee37..29496f38 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -60,7 +60,7 @@ Signoff also needs Git and `gh` with the `gh-signoff` extension. After Copier creates the repository, run `go mod tidy` once, which writes `go.sum` for the recipes. Then commit the repository with Git, since `just update` needs a clean Git tree. Run `lefthook install` once per clone, so `just check` runs before each push. GitHub runs it again for each push to `main` and each pull request. - `just check`: every check. It runs `go mod tidy -diff`, `gofmt`, `go vet`, and `go test`, whose test kit installs the extension into scratch folders -- `just finish`: the exit of the loop. `extension-repository.md` in the pinned contract lists what it checks +- `just finish`: the exit of the loop. `finish-gate.md` in the pinned contract lists what it checks - `just signoff`: runs `just finish` and `just check` on the committed SHA in a detached scratch worktree. Then it writes a receipt outside the repository, and signs that exact SHA - `just install`: installs the extension into Nicely's extensions folder, in place of its previous install - `just uninstall`: removes that install, and keeps its keys and grants From 4e658167767796c329a0cc02b190aeb715b09abd Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:32:47 -0400 Subject: [PATCH 05/61] =?UTF-8?q?=E2=9C=A8=20feat:=20template:=20finish=20?= =?UTF-8?q?reads=20a=20ledger=20of=20rounds=20and=20findings=20in=20readin?= =?UTF-8?q?ess.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readiness.md now holds the ledger of a build. ## Rounds records each round, its head, its reviewers, the commits that each review covered, and the result of just finish. ## Order of round 1 records the commit of each step. Each finding keeps a stable ID and gets a state. just finish fails on more than 5 rounds, with a message to stop and to report to the user. It fails on a finding of an earlier round without a state, on a warn or block that is declined, on an open finding, on a commit outside HEAD, and on a last round whose head is not Commit:. It compares the findings with each committed readiness.md, so a finding never leaves the ledger and its severity never goes down. Finish also names the cause of a severity outside its line: the word, the table, or the fenced code. It reads a severity in each part after the header, ignores a byte order mark, and says that it reads the header in the first three lines. Part of #416 and #417. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 50 +++++++- scripts/template-finish-helpers.sh | 19 ++- scripts/template-finish-test.sh | 23 ++-- scripts/template-ledger-test.sh | 108 ++++++++++++++++ scripts/template-signoff-test.sh | 2 +- scripts/template-test.sh | 3 + sdk/extensionrepo/finish.go | 7 +- sdk/extensionrepo/ledger.go | 164 ++++++++++++++++++++++++ sdk/extensionrepo/ledger_findings.go | 103 +++++++++++++++ sdk/extensionrepo/readiness.go | 66 +++++++--- sdk/extensionrepo/readiness_verdict.go | 42 +++--- 11 files changed, 534 insertions(+), 53 deletions(-) create mode 100644 scripts/template-ledger-test.sh create mode 100644 sdk/extensionrepo/ledger.go create mode 100644 sdk/extensionrepo/ledger_findings.go diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index 5b429936..99bdc573 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -9,9 +9,51 @@ This part says what `just finish` checks in an extension repository, as [Extensi - A line of `vision.md`, `spec.md`, or `qa.md` that starts with `Fill:` after its indent, also in fenced code - A line under `## Open lines` - A line under **Limits.** that is no table row with a spec ID, the state `limit`, and the user's acceptance -- A `readiness.md` whose first three lines hold no `Commit: `, no `Reviewer: `, or no verdict `Ready` or `Ready with stated limits` +- A `readiness.md` whose first three lines hold no `Commit: `, no `Reviewer: `, or no verdict `Ready` or `Ready with stated limits`. A byte order mark before them is ignored - A reviewed commit that differs from the working tree outside `readiness.md` - A finding of `readiness.md` without its own `###` heading under `## Findings`, or without one severity: `block`, `warn`, or `advise` -- A severity under `## Findings` outside the line `- **Severity**:` of a finding, such as in a table, in fenced code, or in prose -- `Ready` with a `block` or `warn` finding, or with a limit in `spec.md` -- `Ready with stated limits` with a `block` finding, or with a `warn` whose **Limit** names no accepted limit +- A severity after the first three lines, outside the line `- **Severity**:` of a finding, such as in a table, in fenced code, or in prose. Under `## Findings`, the word severity fails too. The message names the word or the table that caused it +- `Ready` with a `block` or `warn` finding that is not `fixed`, or with a limit in `spec.md` +- `Ready with stated limits` with a `block` finding that is not `fixed`, or with a `warn` that is neither `fixed` nor an accepted `limit` + +## The ledger + +`readiness.md` also holds the ledger of the build of a command, from the intake to "Done when" (#417). +The reviewer of each round updates it, and commits `readiness.md` alone. + +Under `## Rounds`, a table has the columns Round, Head, Change review, Change commits, Readiness review, and Finish. +Each row is one round, numbered from 1. +Head is the full SHA of the reviewed commit. +Change commits names the commits that the change review covered, as `..`. +The readiness review covers Head. +When one pass did both reviews, both cells name the same reviewer. +Finish is `pass` or `fail` for an earlier round, and `pending` for the last round, whose result is this run. + +Under `## Order of round 1`, a table with the columns Step and Commit names the commit of each step: Intake, Spec, Scenario, and Handler. + +Each finding under `## Findings` starts its heading with a stable ID, such as `### R2-F3`. +`R2` is the round that first reported it, and the ID stays in later rounds. +The reviewer of a later round gives each earlier finding one state, in a line `- **State**:`: + +| State | Meaning | Parts | +|---|---|---| +| `open` | Work remains | `- **Spec**:` with its spec ID | +| `fixed` | A test proves the fix | `- **Commit**:` with the fixing commit, and `- **Spec**:` with a line of Failures or Acceptance | +| `limit` | The user accepted the loss | `- **Spec**:` with the ID of an accepted row under **Limits.** | +| `declined` | The author does not fix it | `- **Reason**:` with the reason of the author. Only an `advise` can be declined | + +A finding of the last round can have no state yet. + +`just finish` fails on each of these: + +- No round under `## Rounds`, or a row without its six cells, its number, a full Head, or both reviewers +- More than 5 rounds. The message tells the agent to stop and to report to the user +- A commit of a round or of a fixed finding that is not HEAD or an ancestor of HEAD +- A last round whose Head is not the `Commit:` of `readiness.md` +- A step of round 1 that is missing or out of order, or a handler in the commit of the first scenario +- A finding without an ID, a finding of an earlier round without a state, or a state without its parts +- A finding with the state `open`, even when `## Open lines` holds `None.` +- A `warn` or `block` finding with the state `declined` +- A finding of an earlier committed `readiness.md` that is missing, or whose severity went down + +At 5 rounds, a failing `just finish` also tells the agent to stop and to report each open line and each open finding. diff --git a/scripts/template-finish-helpers.sh b/scripts/template-finish-helpers.sh index bd37c5f8..2dea96bc 100644 --- a/scripts/template-finish-helpers.sh +++ b/scripts/template-finish-helpers.sh @@ -25,7 +25,8 @@ fct_finish_said() { # fct_readiness writes readiness.md for the commit $1, with the verdict $2, the # findings $3 under ## Findings, and the reviewer $4. An empty $3 leaves out -# ## Findings, and an empty $4 leaves out the line Reviewer:. +# ## Findings, and an empty $4 leaves out the line Reviewer:. The ledger +# holds $5 rounds, or 1, and its last round reviewed $1. fct_readiness() { local reviewer="${4-an agent that wrote none of the code}" { @@ -33,5 +34,21 @@ fct_readiness() { [[ -z "${reviewer}" ]] || printf 'Reviewer: %s\n' "${reviewer}" printf 'Verdict: %s\n\n# Readiness review\n' "${2}" [[ -z "${3}" ]] || printf '\n## Findings\n\n%s\n' "${3}" + fct_ledger "${finish}" "${1}" "${5:-1}" } >"${finish}/readiness.md" } + +# fct_ledger prints the ledger of a build of $3 rounds in the repository $1, +# whose last round reviewed $2. Earlier rounds name the first commit, and +# round 1 commits its handler in $2 (#417). +fct_ledger() { + local first i + first="$(git -C "${1}" rev-list --max-parents=0 HEAD)" + printf '\n## Rounds\n\n| Round | Head | Change review | Change commits | Readiness review | Finish |\n|---|---|---|---|---|---|\n' + for ((i = 1; i < ${3}; i++)); do + printf '| %s | %s | agent A | %s..%s | agent B | `fail` |\n' "${i}" "${first}" "${first}" "${first}" + done + printf '| %s | %s | agent A | %s..%s | agent B | `pending` |\n' "${3}" "${2}" "${first}" "${2}" + printf '\n## Order of round 1\n\n| Step | Commit |\n|---|---|\n' + printf '| %s | %s |\n' Intake "${first}" Spec "${first}" Scenario "${first}" Handler "${2}" +} diff --git a/scripts/template-finish-test.sh b/scripts/template-finish-test.sh index ca3b153b..20f29e85 100644 --- a/scripts/template-finish-test.sh +++ b/scripts/template-finish-test.sh @@ -12,9 +12,9 @@ fct_test_finish() { local finish="${2}/finish" scratch="${2}" sha file number local free='this limit has no spec ID, no state limit, and no acceptance' local unfilled='this line of the form is not filled' - local warn=$'### Large files are slow\n\n- **Severity**: `warn`' - local advise=$'### The count could name the longest line\n\n- **Severity**: `advise`' - local hidden=$'### The count could name the longest line\n\n- **Severity**: `advise`\n\n| Output keeps GPS | `warn` |\n| Output keeps the serial number | block |\n\n```\n### Output keeps the place\n\n- **Severity**: `warn`\n```\n\n### Output keeps the name\n\n- Severity: warn\n- **Severity**: `advise`\n\n### Output keeps the date\n\nSeverity: warn.\n\n- **Severity**: `advise`\n\n### Severity: warn\n\n- **Severity**: `advise`\n- **Limit**: `block`' + local warn=$'### R1-F1 Large files are slow\n\n- **Severity**: `warn`' + local advise=$'### R1-F2 The count could name the longest line\n\n- **Severity**: `advise`' + local hidden=$'### R1-F2 The count could name the longest line\n\n- **Severity**: `advise`\n\n| Output keeps GPS | `warn` |\n| Output keeps the serial number | block |\n\n```\n### Output keeps the place\n\n- **Severity**: `warn`\n```\n\n### R1-F3 Output keeps the name\n\n- Severity: warn\n- **Severity**: `advise`\n\n### R1-F4 Output keeps the date\n\nSeverity: warn.\n\n- **Severity**: `advise`\n\n### Severity: warn\n\n- **Severity**: `advise`\n- **Spec**: `block`' local unread=$'- **Severity**: `warn`\n\n### No severity\n\n### Two severities\n\n- **Severity**: `advise`\n- **Severity**: `warn`\n\n### Unknown severity\n\n- **Severity**: minor\n\n### A fenced heading\n\n````\n# a comment\n```\n````\n\n- **Severity**: `block`\n\n### An open fence\n\n```\n- **Severity**: `advise`' cp -R "${1}" "${finish}" [[ "$(just --justfile "${finish}/justfile" --show finish | sed '/^#/d')" == $'finish:\n go run github.com/pascalandy/nicely/sdk/extensionrepo/cmd/extensionrepo finish' ]] || @@ -62,7 +62,7 @@ fct_test_finish() { git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'independent readiness review' fct_finish 0 - fct_finish_said 'finish: no line is open' + fct_finish_said 'finish: no line or finding is open' fct_edit "${finish}/spec.md" '**Limits.** None.' $'**Limits.**\n\n| ID | State | Limit | Accepted |\n|---|---|---|---|\n| `large-file` | `open` | A file over 1 GB takes more than 30 s to count | The user, 2026-10-30, https://example.com/pull/1 |' fct_finish 1 @@ -84,7 +84,7 @@ fct_test_finish() { fct_finish 1 fct_finish_said 'readiness.md:9: this warn names no limit that the user accepted' fct_finish_said 'readiness.md:13: the verdict is Ready with stated limits, but this finding has the severity block' - fct_readiness "${sha}" 'Ready with stated limits' "${warn}"$'\n- **Limit**: `small-file`' + fct_readiness "${sha}" 'Ready with stated limits' "${warn}"$'\n- **State**: `limit`\n- **Spec**: `small-file`' fct_finish 1 fct_finish_said 'readiness.md:9: this warn names no limit that the user accepted' @@ -118,8 +118,11 @@ fct_test_finish() { # A severity outside the line - **Severity**: of a finding fails it, even after a valid finding (#413). fct_readiness "${sha}" 'Ready with stated limits' "${hidden}" fct_finish 1 - for number in 13 14 19 24 29 33 36; do - fct_finish_said "readiness.md:${number}: this line names a severity outside the line - **Severity**: of a finding" + fct_finish_said 'readiness.md:13: this table names the severity warn' + fct_finish_said 'readiness.md:14: this table names the severity block' + fct_finish_said 'readiness.md:19: this fenced code names the severity warn' + for number in 24 29 33 36; do + fct_finish_said "readiness.md:${number}: this line names the severity" done # Finish reads Commit:, Reviewer:, and Verdict: only in the first lines, never in fenced code (#413). @@ -128,13 +131,13 @@ fct_test_finish() { fct_finish_said 'readiness.md: it names no reviewer' fct_readiness '' 'Ready with stated limits' $'```\nCommit: '"${sha}"$'\n```\n\nNone.' fct_finish 1 - fct_finish_said 'readiness.md: it holds no line Commit: or Verdict: ' + fct_finish_said 'readiness.md: its first three lines hold no line Commit: or Verdict: ' - fct_readiness "${sha}" 'Ready with stated limits' "${warn}"$'\n- **Limit**: `large-file`\n\n'"${advise}"$'\n\n## Severity assessment\n\nEach finding names its severity.' + fct_readiness "${sha}" 'Ready with stated limits' "${warn}"$'\n- **State**: `limit`\n- **Spec**: `large-file`\n\n'"${advise}"$'\n\n## Severity assessment\n\nEach finding names its severity.' git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'readiness review' fct_finish 0 - fct_finish_said 'finish: no line is open' + fct_finish_said 'finish: no line or finding is open' # A change after the review needs a new review. printf 'A change after the review.\n' >>"${finish}/README.md" diff --git a/scripts/template-ledger-test.sh b/scripts/template-ledger-test.sh new file mode 100644 index 00000000..a0cfef0b --- /dev/null +++ b/scripts/template-ledger-test.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Sourced by template-test.sh to prove the ledger of readiness.md, which +# just finish reads (#417), and the forms of readiness.md of #416. It works on +# a copy of the rendered repository, so the other tests keep their state. +# Markdown backticks in the fixture text are literal. +# shellcheck disable=SC2016 + +fct_test_ledger() { + local finish="${2}/ledger" scratch="${2}" file first round2 head stray valid + local advise=$'### R1-F1 The count could name the longest line\n\n- **Severity**: `advise`' + local declined=$'\n- **State**: `declined`\n- **Reason**: the answer stays small' + local warn=$'### R2-F1 Long lines are cut\n\n- **Severity**: `warn`' + local lowered=$'### R2-F1 Long lines are cut\n\n- **Severity**: `advise`' + cp -R "${1}" "${finish}" + for file in vision.md spec.md qa.md; do + grep -v '^Fill:' "${finish}/${file}" >"${scratch}/form.md" + mv "${scratch}/form.md" "${finish}/${file}" + done + git -C "${finish}" commit -qam 'fill the forms' + first="$(git -C "${finish}" rev-list --max-parents=0 HEAD)" + + # Round 1 reports an advise, and round 2 declines it and reports a warn. + # Their committed reviews are the history of round 3. + fct_readiness "$(git -C "${finish}" rev-parse HEAD)" 'Needs work' "${advise}" + git -C "${finish}" add readiness.md + git -C "${finish}" commit -qm 'readiness review of round 1' + printf 'Round 2.\n' >>"${finish}/README.md" + git -C "${finish}" commit -qam 'round 2' + round2="$(git -C "${finish}" rev-parse HEAD)" + fct_readiness "${round2}" 'Needs work' "${advise}${declined}"$'\n\n'"${warn}" 'agent B' 2 + git -C "${finish}" add readiness.md + git -C "${finish}" commit -qm 'readiness review of round 2' + printf 'Round 3.\n' >>"${finish}/README.md" + git -C "${finish}" commit -qam 'round 3' + head="$(git -C "${finish}" rev-parse HEAD)" + + # A warn of round 2 that the ledger of round 3 does not record fails finish. + fct_readiness "${head}" Ready "${advise}${declined}" 'agent B' 3 + fct_finish 1 + fct_finish_said 'readiness.md: the finding R2-F1 of the review in' + # A finding keeps its highest severity, so a warn never closes as a declined advise. + fct_readiness "${head}" Ready "${advise}${declined}"$'\n\n'"${lowered}${declined}" 'agent B' 3 + fct_finish 1 + fct_finish_said 'the finding R2-F1 had the severity warn' + fct_readiness "${head}" Ready "${advise}${declined}"$'\n\n'"${warn}${declined}" 'agent B' 3 + fct_finish 1 + fct_finish_said 'the finding R2-F1 has the severity warn, and only an advise can be declined' + fct_readiness "${head}" Ready "${advise}${declined}"$'\n\n'"${warn}" 'agent B' 3 + fct_finish 1 + fct_finish_said 'the finding R2-F1 of round 2 has no state' + # None. under ## Open lines does not close a finding that the ledger records as open. + fct_readiness "${head}" 'Ready with stated limits' "${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `open`\n- **Spec**: `long-lines`' 'agent B' 3 + fct_finish 1 + fct_finish_said 'the finding R2-F1 is open, even if ## Open lines holds None.' + fct_readiness "${head}" Ready "${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `fixed`\n- **Spec**: `long-lines`' 'agent B' 3 + fct_finish 1 + fct_finish_said 'this fixed finding names no commit of HEAD' + fct_finish_said 'this fixed finding names no line of Failures or Acceptance in spec.md' + valid="${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `fixed`\n- **Commit**: '"${head}"$'\n- **Spec**: `bad-input`' + fct_readiness "${head}" Ready "${valid}" 'agent B' 3 + git -C "${finish}" add readiness.md + git -C "${finish}" commit -qm 'readiness review of round 3' + fct_finish 0 + + # A build has at most 5 rounds, and finish tells the agent to stop. + fct_readiness "${head}" Ready "${valid}" 'agent B' 6 + fct_finish 1 + fct_finish_said 'readiness.md: the ledger holds 6 rounds, and a build has at most 5' + fct_finish_said 'this build used its 5 rounds. Stop, and report each open line and each open finding to the user' + fct_readiness "${head}" Ready "${valid}" 'agent B' 3 + stray="$(git -C "${finish}" commit-tree -m stray "${head}^{tree}")" + fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${first}..${head} | agent B | \`pending\` |" "| 3 | ${head} | agent A | ${stray}..${head} | agent B | \`pending\` |" + fct_finish 1 + fct_finish_said "the commit \"${stray}\" is not an ancestor of HEAD" + fct_readiness "${head}" Ready "${valid}" 'agent B' 3 + fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${first}..${head} | agent B | \`pending\` |" "| 3 | ${round2} | agent A | ${first}..${head} | agent B | \`pending\` |" + fct_finish 1 + fct_finish_said "the last round names the head ${round2}, and the line Commit: names ${head}" + # Round 1 commits its first scenario before its handler. + fct_readiness "${head}" Ready "${valid}" 'agent B' 3 + fct_edit "${finish}/readiness.md" "| Scenario | ${first} |" "| Scenario | ${head} |" + fct_finish 1 + fct_finish_said 'the handler and the first scenario are in one commit' + printf 'Commit: %s\nReviewer: agent B\nVerdict: Ready\n\n## Findings\n\n%s\n' "${head}" "${valid}" >"${finish}/readiness.md" + fct_finish 1 + fct_finish_said 'readiness.md: it holds no round under ## Rounds' + + # The word severity in prose, or a table of severities, fails with its cause (#416). + fct_readiness "${head}" Ready "${advise}"$'\n- **Evidence**: I kept the severity low'"${declined}"$'\n\n| Finding | Grade |\n|---|---|\n| R1-F1 | advise |' 'agent B' 3 + fct_finish 1 + fct_finish_said 'this line names the word severity outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity' + fct_finish_said 'this table names the severity advise' + # A warn in prose beside an advise, or in a table of a part after ## Findings, fails (#416). + fct_readiness "${head}" Ready "${valid}"$'\n\nThis is a warn.\n\n## Notes\n\n| Long lines kept | `warn` |' 'agent B' 3 + fct_finish 1 + fct_finish_said 'this line names the severity warn outside the line - **Severity**: of a finding' + fct_finish_said 'this table names the severity warn' + # Finish reads the header in the first three lines, after a byte order mark (#416). + fct_readiness "${head}" Ready "${valid}" 'agent B' 3 + printf '# Readiness review\n' | cat - "${finish}/readiness.md" >"${scratch}/readiness.md" + mv "${scratch}/readiness.md" "${finish}/readiness.md" + fct_finish 1 + fct_finish_said 'readiness.md: its first three lines hold no line Commit: or Verdict: ' + git -C "${finish}" checkout -q readiness.md + printf '\357\273\277' | cat - "${finish}/readiness.md" >"${scratch}/readiness.md" + mv "${scratch}/readiness.md" "${finish}/readiness.md" + fct_finish 0 +} diff --git a/scripts/template-signoff-test.sh b/scripts/template-signoff-test.sh index 5ba61ee6..a0dbbc15 100644 --- a/scripts/template-signoff-test.sh +++ b/scripts/template-signoff-test.sh @@ -30,7 +30,7 @@ SCRIPT mv "${scratch}/form.md" "${ready}/${file}" done git -C "${ready}" commit -qam 'fill the forms' - printf 'Commit: %s\nReviewer: an agent that wrote none of the code\nVerdict: Ready\n\n## Findings\n\nNone.\n' "$(git -C "${ready}" rev-parse HEAD)" >"${ready}/readiness.md" + finish="${ready}" fct_readiness "$(git -C "${ready}" rev-parse HEAD)" Ready None. git -C "${ready}" add readiness.md git -C "${ready}" commit -qm 'readiness review' sha="$(git -C "${ready}" rev-parse HEAD)" diff --git a/scripts/template-test.sh b/scripts/template-test.sh index 7afb1805..76a899de 100644 --- a/scripts/template-test.sh +++ b/scripts/template-test.sh @@ -71,6 +71,7 @@ fct_repository() { fct_commit "rendered from v0.0.1" (cd "${repo}" && just check) fct_test_finish "${repo}" "${scratch}" + fct_test_ledger "${repo}" "${scratch}" fct_test_example_notices "${repo}" "${scratch}" "${slug}" export XDG_CONFIG_HOME="${scratch}/config" XDG_STATE_HOME="${scratch}/state" XDG_DATA_HOME="${data}" NCLY_NO_INPUT=1 (cd "${repo}" && just install) @@ -135,6 +136,8 @@ fct_main() { source "${root}/scripts/template-signoff-test.sh" # shellcheck source=scripts/template-finish-test.sh source "${root}/scripts/template-finish-test.sh" + # shellcheck source=scripts/template-ledger-test.sh + source "${root}/scripts/template-ledger-test.sh" # A Git hook or a caller can export GIT_DIR. The scratch repositories must see their own. while IFS= read -r variable; do unset "${variable}"; done < <(git rev-parse --local-env-vars) # just update runs the Copier that the template pins, and so does this test. diff --git a/sdk/extensionrepo/finish.go b/sdk/extensionrepo/finish.go index cecf6f64..a3fda262 100644 --- a/sdk/extensionrepo/finish.go +++ b/sdk/extensionrepo/finish.go @@ -16,14 +16,17 @@ import ( func finishRepository() int { problems, spec := formProblems() specs, limits := specProblems(spec) - problems = append(append(problems, specs...), readinessProblems(limits)...) + problems = append(append(problems, specs...), readinessProblems(spec, limits)...) for _, problem := range problems { fmt.Fprintln(os.Stderr, problem) } if len(problems) != 0 { + if ledgerRounds() >= maxRounds { + fmt.Fprintf(os.Stderr, "finish: this build used its %d rounds. Stop, and report each open line and each open finding to the user. An answer that lets you choose starts no round\n", maxRounds) + } return report("finish", errors.New("the loop of the nicely-extension skill is not done")) } - fmt.Println("finish: no line is open, the user accepted each limit, and readiness.md holds a passing verdict for this commit") + fmt.Println("finish: no line or finding is open, the user accepted each limit, and readiness.md holds a passing verdict and ledger for this commit") return 0 } diff --git a/sdk/extensionrepo/ledger.go b/sdk/extensionrepo/ledger.go new file mode 100644 index 00000000..27b78b39 --- /dev/null +++ b/sdk/extensionrepo/ledger.go @@ -0,0 +1,164 @@ +package extensionrepo + +import ( + "fmt" + "os" + "os/exec" + "regexp" + "slices" + "strings" +) + +// maxRounds bounds the rounds of one build of a command, from the intake to +// "Done when" (#417). +const maxRounds = 5 + +// The ledger of readiness.md records each round of the build in a table +// under ## Rounds. It records the commit of each step of round 1 under +// ## Order of round 1. Each finding under ## Findings keeps its ID from the +// round that first reported it, and gets a state (#417). +var ( + roundColumns = []string{"Round", "Head", "Change review", "Change commits", "Readiness review", "Finish"} + orderSteps = []string{"Intake", "Spec", "Scenario", "Handler"} + ledgerSHA = regexp.MustCompile("^`?([0-9a-f]{7,40})`?$") + ledgerFullSHA = regexp.MustCompile("^`?([0-9a-f]{40})`?$") + severityRank = map[string]int{"advise": 1, "warn": 2, "block": 3} +) + +type round struct { + line int + cells []string +} + +// ledgerProblems reports each rule of the ledger that readiness.md breaks. +// reviewed is the full SHA of its line Commit:. +func ledgerProblems(readiness, reviewed string, findings []finding, spec string, limits []limit) []string { + const fix = "Fix: give readiness.md a table under ## Rounds with the columns " + + "Round, Head, Change review, Change commits, Readiness review, and Finish, one row for each round" + var problems []string + var rounds []round + order := map[string]round{} + for _, l := range SpecLines(readiness) { + switch { + case l.Part == "Rounds" && l.Kind == SpecHeader && !slices.Equal(l.Cells, roundColumns): + problems = append(problems, fmt.Sprintf("readiness.md:%d: the table of ## Rounds has other columns. %s", l.Number, fix)) + case l.Part == "Rounds" && l.Kind == SpecRow: + rounds = append(rounds, round{l.Number, l.Cells}) + case l.Part == "Order of round 1" && l.Kind == SpecRow && len(l.Cells) == 2: + order[l.Cells[0]] = round{l.Number, l.Cells} + } + } + if len(rounds) == 0 { + return append(problems, "readiness.md: it holds no round under ## Rounds. "+fix) + } + if len(rounds) > maxRounds { + problems = append(problems, fmt.Sprintf("readiness.md: the ledger holds %d rounds, and a build has at most %d. "+ + "Fix: stop, and report each open line and each open finding to the user. Do not start another round", len(rounds), maxRounds)) + } + for i, r := range rounds { + problems = append(problems, roundProblems(r, i+1, i == len(rounds)-1)...) + } + if last := rounds[len(rounds)-1]; len(last.cells) == len(roundColumns) { + if head, ok := ledgerCommit(last.cells[1]); ok && head != reviewed { + problems = append(problems, fmt.Sprintf("readiness.md:%d: the last round names the head %s, and the line Commit: names %s. "+ + "Fix: name the reviewed commit in both", last.line, last.cells[1], reviewed)) + } + } + problems = append(problems, orderProblems(order)...) + problems = append(problems, stateProblems(findings, len(rounds), spec, limits)...) + return append(problems, historyProblems(findings)...) +} + +// roundProblems checks one row of ## Rounds, the round number of its place. +func roundProblems(r round, number int, last bool) []string { + at := fmt.Sprintf("readiness.md:%d: ", r.line) + if len(r.cells) != len(roundColumns) { + return []string{at + "this round has no cell for each column of ## Rounds. Fix: fill Round, Head, Change review, Change commits, Readiness review, and Finish"} + } + var problems []string + if r.cells[0] != fmt.Sprint(number) { + problems = append(problems, fmt.Sprintf("%sthis row is round %d, but its Round cell holds %q. Fix: number the rounds 1, 2, 3, in order", at, number, r.cells[0])) + } + if !ledgerFullSHA.MatchString(r.cells[1]) { + problems = append(problems, at+"its Head is no full SHA. Fix: write the 40 characters of the reviewed commit") + } + commits := []string{r.cells[1]} + for _, c := range strings.Split(r.cells[3], "..") { + commits = append(commits, strings.TrimSpace(c)) + } + for _, c := range commits { + if _, ok := ledgerCommit(c); !ok { + problems = append(problems, fmt.Sprintf("%sthe commit %q is not an ancestor of HEAD. Fix: name the commits of this branch that the round reviewed, as ..", at, c)) + } + } + if r.cells[2] == "" || r.cells[4] == "" { + problems = append(problems, at+"it names no reviewer of the change review or of the readiness review. Fix: name the agent that ran each review, which wrote none of its code") + } + finish, want := strings.Trim(r.cells[5], "`"), []string{"pass", "fail"} + if last { + want = []string{"pending"} + } + if !slices.Contains(want, finish) { + problems = append(problems, fmt.Sprintf("%sits Finish is %q. Fix: write the result of just finish for the round, pass or fail, and pending for the last round, whose result is this run", at, finish)) + } + return problems +} + +// orderProblems checks that round 1 recorded the intake, the spec, the +// first scenario, and the handler, each in a commit after the one before. +// The scenario precedes the handler, so it could fail first. +func orderProblems(order map[string]round) []string { + const fix = "Fix: give readiness.md a table under ## Order of round 1 with the columns Step and Commit, and the rows Intake, Spec, Scenario, and Handler" + var problems []string + previous := "" + for _, step := range orderSteps { + r, found := order[step] + if !found { + problems = append(problems, fmt.Sprintf("readiness.md: ## Order of round 1 gives no commit for the step %s. %s", step, fix)) + previous = "" + continue + } + commit, ok := ledgerCommit(r.cells[1]) + switch { + case !ok: + problems = append(problems, fmt.Sprintf("readiness.md:%d: the commit of the step %s is not an ancestor of HEAD. %s", r.line, step, fix)) + case previous != "" && !ancestor(previous, commit): + problems = append(problems, fmt.Sprintf("readiness.md:%d: the step %s comes before the step that precedes it in round 1. Fix: do the steps of round 1 in the order of the skill", r.line, step)) + case step == "Handler" && commit == previous: + problems = append(problems, fmt.Sprintf("readiness.md:%d: the handler and the first scenario are in one commit, so the scenario never failed first. Fix: commit the failing scenario before the handler", r.line)) + } + previous = commit + } + return problems +} + +// ledgerCommit resolves a commit of the ledger. It returns its full SHA, and +// whether it is HEAD or an ancestor of HEAD. +func ledgerCommit(cell string) (string, bool) { + m := ledgerSHA.FindStringSubmatch(strings.TrimSpace(cell)) + if m == nil { + return "", false + } + out, err := exec.Command("git", "rev-parse", "--verify", "--quiet", m[1]+"^{commit}").Output() + sha := strings.TrimSpace(string(out)) + return sha, err == nil && ancestor(sha, "HEAD") +} + +func ancestor(commit, of string) bool { + return exec.Command("git", "merge-base", "--is-ancestor", commit, of).Run() == nil +} + +// ledgerRounds counts the rounds of the ledger, for the note of finish. +func ledgerRounds() int { + data, err := os.ReadFile("readiness.md") + if err != nil { + return 0 + } + n := 0 + for _, l := range SpecLines(string(data)) { + if l.Part == "Rounds" && l.Kind == SpecRow { + n++ + } + } + return n +} diff --git a/sdk/extensionrepo/ledger_findings.go b/sdk/extensionrepo/ledger_findings.go new file mode 100644 index 00000000..d7e50049 --- /dev/null +++ b/sdk/extensionrepo/ledger_findings.go @@ -0,0 +1,103 @@ +package extensionrepo + +import ( + "fmt" + "os/exec" + "slices" + "strconv" + "strings" +) + +// stateProblems checks the state of each finding in the build of n rounds. +// A new finding of round n can have no state yet. +func stateProblems(findings []finding, n int, spec string, limits []limit) []string { + var proven []string + for _, l := range SpecLines(spec) { + if l.ID != "" && (l.Part == "Failures" || l.Part == "Acceptance") { + proven = append(proven, l.ID) + } + } + var problems []string + seen := map[string]bool{} + for _, f := range findings { + at := fmt.Sprintf("readiness.md:%d: ", f.line) + m := readinessID.FindStringSubmatch(f.id) + if m == nil { + problems = append(problems, at+"this finding has no ID. Fix: start its heading with R, the round that first reported it, then -F and its number, such as ### R2-F1") + continue + } + first, _ := strconv.Atoi(m[2]) + switch { + case seen[f.id]: + problems = append(problems, at+"another finding has the ID "+f.id+". Fix: give each finding its own ID, and keep it in later rounds") + case first > n: + problems = append(problems, fmt.Sprintf("%sthe finding %s is of round %d, after the last round %d of ## Rounds. Fix: correct its ID or the rounds", at, f.id, first, n)) + } + seen[f.id] = true + switch f.state { + case "": + if first < n { + problems = append(problems, fmt.Sprintf("%sthe finding %s of round %d has no state. Fix: the reviewer of this round gives it the state open, fixed, limit, or declined", at, f.id, first)) + } + case "open": + problems = append(problems, fmt.Sprintf("%sthe finding %s is open, even if ## Open lines holds None. Fix: prove its spec ID in a round, or ask the user to accept it as a limit", at, f.id)) + case "fixed": + if commit, ok := ledgerCommit(f.commit); !ok || commit == "" { + problems = append(problems, at+"this fixed finding names no commit of HEAD. Fix: name the commit that fixed it in a line - **Commit**: ") + } + if !slices.Contains(proven, f.spec) { + problems = append(problems, at+"this fixed finding names no line of Failures or Acceptance in spec.md. Fix: name the spec ID that its test proves in a line - **Spec**: ``") + } + case "limit": + if !slices.ContainsFunc(limits, func(l limit) bool { return l.id == f.spec && l.accepted }) { + problems = append(problems, at+"this limit names no limit that the user accepted in spec.md. Fix: name its row under **Limits.** in a line - **Spec**: ``, after the user accepts it") + } + case "declined": + if f.severity != "advise" { + problems = append(problems, fmt.Sprintf("%sthe finding %s has the severity %s, and only an advise can be declined. Fix: fix it, or ask the user to accept it as a limit", at, f.id, f.severity)) + } + if f.why == "" { + problems = append(problems, at+"this declined finding gives no reason. Fix: write the reason of the author in a line - **Reason**:") + } + default: + problems = append(problems, fmt.Sprintf("%sthe state %q is not open, fixed, limit, or declined", at, f.state)) + } + } + return problems +} + +// historyProblems compares the findings with each committed readiness.md of +// this branch. A finding keeps its ID in each later review, and its severity +// never goes down, so no loss leaves the ledger. +func historyProblems(findings []finding) []string { + revisions, err := exec.Command("git", "log", "--format=%H", "HEAD", "--", "readiness.md").Output() + if err != nil { + return []string{"readiness.md: git log: " + err.Error()} + } + current := map[string]finding{} + for _, f := range findings { + current[f.id] = f + } + var problems []string + reported := map[string]bool{} + for _, revision := range strings.Fields(string(revisions)) { + old, err := exec.Command("git", "show", revision+":readiness.md").Output() + if err != nil { + continue + } + earlier, _ := readinessFindings(strings.TrimPrefix(string(old), "\ufeff")) + for _, e := range earlier { + now, kept := current[e.id] + switch { + case e.id == "" || reported[e.id]: + case !kept: + reported[e.id] = true + problems = append(problems, fmt.Sprintf("readiness.md: the finding %s of the review in %.12s is missing. Fix: keep each earlier finding under ## Findings, with its ID and its state", e.id, revision)) + case severityRank[now.severity] < severityRank[e.severity]: + reported[e.id] = true + problems = append(problems, fmt.Sprintf("readiness.md:%d: the finding %s had the severity %s in %.12s, and now has %s. Fix: keep its highest severity. A warn or block closes only as fixed or limit", now.line, e.id, e.severity, revision, now.severity)) + } + } + } + return problems +} diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index 39a300e5..556bffdf 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -7,13 +7,13 @@ import ( "strings" ) -// A finding is a ### heading under ## Findings of readiness.md. Its bullet -// - **Severity**: grades it. For a warn, its bullet - **Limit**: names the -// spec ID of the limit that the user accepted for it (#409). +// A finding is a ### heading under ## Findings of readiness.md, which starts +// with its ID, such as R2-F3. Its bullet - **Severity**: grades it. Its +// bullets - **State**:, - **Spec**:, - **Commit**:, and - **Reason**: give +// its state in the ledger (#417). type finding struct { - line int - severity string - limit string + line int + id, severity, state, spec, commit, why string } var ( @@ -21,13 +21,30 @@ var ( readinessReviewer = regexp.MustCompile(`(?m)^Reviewer:[ \t]*\S`) readinessVerdict = regexp.MustCompile(`(?m)^Verdict: (.+?)\s*$`) readinessHeading = regexp.MustCompile(`^(#{1,6})\s+(.+?)\s*$`) - readinessPart = regexp.MustCompile(`^\s*[-*+]\s+\*\*(Severity|Limit)(?:\*\*:|:\*\*)\s*(.*?)\s*$`) - // readinessSeverity finds a severity in a form that finish cannot grade. - // Such a form is the word severity, a severity in code, or a severity - // cell of a table (#413). - readinessSeverity = regexp.MustCompile(`(?i)\bseverity\b|` + "`(?:block|warn|advise)`" + `|\|\s*(?:block|warn|advise)\s*(?:\||$)`) + readinessPart = regexp.MustCompile(`^\s*[-*+]\s+\*\*(Severity|State|Spec|Commit|Reason)(?:\*\*:|:\*\*)\s*(.*?)\s*$`) + readinessID = regexp.MustCompile(`^(R([1-9][0-9]*)-F[1-9][0-9]*)\b`) + // readinessSeverity finds a severity in a form that finish cannot grade, + // such as prose, code, or a table, anywhere after the header (#413, #416). + readinessSeverity = regexp.MustCompile(`(?i)\b(block|warn|advise)\b`) + // readinessWord finds the word severity under ## Findings (#413). + readinessWord = regexp.MustCompile(`(?i)\bseverity\b`) ) +// severityCause names why a line that names a severity fails finish, so the +// reviewer sees the word or the table that caused it (#416). +func severityCause(text, fence string) string { + const fix = "Fix: grade each finding only in its line - **Severity**:, and rephrase this line without " + switch m := readinessSeverity.FindString(text); { + case fence != "": + return fmt.Sprintf("this fenced code names the severity %s. %sthe word %s", strings.ToLower(m), fix, strings.ToLower(m)) + case strings.HasPrefix(strings.TrimSpace(text), "|") && m != "": + return fmt.Sprintf("this table names the severity %s. Fix: remove the table, because each finding gives its severity in its line - **Severity**:", strings.ToLower(m)) + case m != "": + return fmt.Sprintf("this line names the severity %s outside the line - **Severity**: of a finding. %sthe word %s", strings.ToLower(m), fix, strings.ToLower(m)) + } + return "this line names the word severity outside the line - **Severity**: of a finding. " + fix + "the word severity" +} + // readinessFindings reads the findings of readiness.md, outside fenced code. // It reports each finding that finish cannot grade, so a finding in another // form never passes as no finding. Under ## Findings, it also reports each @@ -48,9 +65,11 @@ func readinessFindings(readiness string) ([]finding, []string) { m, heading := readinessPart.FindStringSubmatch(text), readinessHeading.FindStringSubmatch(text) // A heading of level 1 or 2 closes ## Findings, so it is no line of a finding. section := fence == "" && heading != nil && len(heading[1]) <= 2 - bullet := fence == "" && m != nil && m[1] == "Severity" - if part > 0 && !section && !bullet && readinessSeverity.MatchString(text) { - problems = append(problems, fmt.Sprintf("readiness.md:%d: this line names a severity outside the line - **Severity**: of a finding. %s", i+1, fix)) + bullet := part > 0 && fence == "" && m != nil && m[1] == "Severity" + // The first three lines are the header. A severity elsewhere fails, + // also under another part after ## Findings (#416). + if i >= 3 && !bullet && (readinessSeverity.MatchString(text) || part > 0 && !section && readinessWord.MatchString(text)) { + problems = append(problems, fmt.Sprintf("readiness.md:%d: %s", i+1, severityCause(text, fence))) } switch { case fence != "": @@ -70,7 +89,11 @@ func readinessFindings(readiness string) ([]finding, []string) { part, seen = i+1, true } case level == 3 && part > 0: - findings = append(findings, finding{line: i + 1}) + f := finding{line: i + 1} + if id := readinessID.FindStringSubmatch(heading[2]); id != nil { + f.id = id[1] + } + findings = append(findings, f) current = len(findings) - 1 } continue @@ -85,9 +108,18 @@ func readinessFindings(readiness string) ([]finding, []string) { } value := strings.Trim(strings.TrimSuffix(m[2], "."), "`") switch { - case m[1] == "Limit": + case m[1] != "Severity": if current >= 0 { - findings[current].limit = value + switch f := &findings[current]; m[1] { + case "State": + f.state = value + case "Spec": + f.spec = value + case "Commit": + f.commit = value + case "Reason": + f.why = value + } } case current < 0: problems = append(problems, fmt.Sprintf("readiness.md:%d: this severity has no finding of its own. %s", i+1, fix)) diff --git a/sdk/extensionrepo/readiness_verdict.go b/sdk/extensionrepo/readiness_verdict.go index 946a121f..8c1c6f35 100644 --- a/sdk/extensionrepo/readiness_verdict.go +++ b/sdk/extensionrepo/readiness_verdict.go @@ -10,10 +10,10 @@ import ( // readinessProblems reports a readiness.md without a reviewer or a passing // verdict, or whose commit is not the code of the working tree. The verdict -// must follow from the findings and from the limits of spec.md (#409). Only -// readiness.md may differ from that commit, so the review can be committed -// after it. -func readinessProblems(limits []limit) []string { +// must follow from the findings, their states, and the limits of spec.md +// (#409, #417). Only readiness.md may differ from that commit, so the review +// can be committed after it. Then the ledger of the build must hold. +func readinessProblems(spec string, limits []limit) []string { const fix = "Fix: ask an agent that did not write the code to run the readiness review of the nicely-extension skill, then start readiness.md with the lines Commit: , Reviewer: , and Verdict: " const verdictFix = "Fix: write the verdict that the findings and the limits give, as the readiness review of the nicely-extension skill says" data, err := os.ReadFile("readiness.md") @@ -24,21 +24,26 @@ func readinessProblems(limits []limit) []string { } // readiness.md starts with the lines Commit:, Reviewer:, and Verdict:, // as the skill says. So finish never reads them in fenced code (#413). - lines := strings.SplitN(string(data), "\n", 4) + // An editor can write a byte order mark before them (#416). + readiness := strings.TrimPrefix(string(data), "\ufeff") + lines := strings.SplitN(readiness, "\n", 4) header := strings.Join(lines[:min(3, len(lines))], "\n") commit, verdict := readinessCommit.FindStringSubmatch(header), readinessVerdict.FindStringSubmatch(header) if commit == nil || verdict == nil { - return []string{"readiness.md: it holds no line Commit: or Verdict: . " + fix} + return []string{"readiness.md: its first three lines hold no line Commit: or Verdict: . " + fix} } - findings, problems := readinessFindings(string(data)) + findings, problems := readinessFindings(readiness) if !readinessReviewer.MatchString(header) { problems = append(problems, "readiness.md: it names no reviewer. "+fix) } + accepted := func(f finding) bool { + return f.state == "limit" && slices.ContainsFunc(limits, func(l limit) bool { return l.id == f.spec && l.accepted }) + } switch v := verdict[1]; v { case "Ready": for _, f := range findings { - if f.severity == "block" || f.severity == "warn" { - problems = append(problems, fmt.Sprintf("readiness.md:%d: the verdict is Ready, but this finding has the severity %s. %s", f.line, f.severity, verdictFix)) + if (f.severity == "block" || f.severity == "warn") && f.state != "fixed" { + problems = append(problems, fmt.Sprintf("readiness.md:%d: the verdict is Ready, but this finding has the severity %s and is not fixed. %s", f.line, f.severity, verdictFix)) } } for _, l := range limits { @@ -47,10 +52,11 @@ func readinessProblems(limits []limit) []string { case "Ready with stated limits": for _, f := range findings { switch { + case f.state == "fixed": case f.severity == "block": - problems = append(problems, fmt.Sprintf("readiness.md:%d: the verdict is Ready with stated limits, but this finding has the severity block. %s", f.line, verdictFix)) - case f.severity == "warn" && !slices.ContainsFunc(limits, func(l limit) bool { return l.id == f.limit && l.accepted }): - problems = append(problems, fmt.Sprintf("readiness.md:%d: this warn names no limit that the user accepted in spec.md. Fix: name the spec ID of its accepted limit in a line - **Limit**: ``, or write the verdict Needs work", f.line)) + problems = append(problems, fmt.Sprintf("readiness.md:%d: the verdict is Ready with stated limits, but this finding has the severity block and is not fixed. %s", f.line, verdictFix)) + case f.severity == "warn" && !accepted(f): + problems = append(problems, fmt.Sprintf("readiness.md:%d: this warn names no limit that the user accepted in spec.md. Fix: give it the state `limit` and the spec ID of its accepted limit in a line - **Spec**: ``, or write the verdict Needs work", f.line)) } } case "Needs work": @@ -58,11 +64,11 @@ func readinessProblems(limits []limit) []string { default: problems = append(problems, fmt.Sprintf("readiness.md: the verdict %q is not Ready, Ready with stated limits, or Needs work. %s", v, fix)) } - sha := commit[1] - if _, err := updateOutput("git", "rev-parse", "--verify", "--quiet", sha+"^{commit}"); err != nil { - return append(problems, fmt.Sprintf("readiness.md: Git knows no commit %s. %s", sha, fix)) + sha, err := updateOutput("git", "rev-parse", "--verify", "--quiet", commit[1]+"^{commit}") + if err != nil { + return append(problems, fmt.Sprintf("readiness.md: Git knows no commit %s. %s", commit[1], fix)) } - changed, err := updateOutput("git", "diff", "--name-only", sha, "--") + changed, err := updateOutput("git", "diff", "--name-only", commit[1], "--") if err != nil { return append(problems, "readiness.md: "+err.Error()) } @@ -77,7 +83,7 @@ func readinessProblems(limits []limit) []string { } } if len(files) != 0 { - problems = append(problems, fmt.Sprintf("readiness.md: the review is of %s, and these files changed since: %s. Fix: run the readiness review again on this commit", sha, strings.Join(files, ", "))) + problems = append(problems, fmt.Sprintf("readiness.md: the review is of %s, and these files changed since: %s. Fix: run the readiness review again on this commit", commit[1], strings.Join(files, ", "))) } - return problems + return append(problems, ledgerProblems(readiness, strings.TrimSpace(string(sha)), findings, spec, limits)...) } From b22166d5071d5ac32bc6e78338268d9627a61cac Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:33:08 -0400 Subject: [PATCH 06/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20template?= =?UTF-8?q?:=20move=20the=20readiness=20review=20of=20the=20skill=20to=20i?= =?UTF-8?q?ts=20own=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The skill nicely-extension nears the size limit. Its readiness review moves to readiness-review.md beside it, with no change of rule. Co-Authored-By: Claude Opus 5.5 --- .../.agents/skills/nicely-extension/SKILL.md | 45 +----------------- .../nicely-extension/readiness-review.md | 47 +++++++++++++++++++ 2 files changed, 49 insertions(+), 43 deletions(-) create mode 100644 template/.agents/skills/nicely-extension/readiness-review.md diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index 83df37db..2fc3ba82 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -107,46 +107,5 @@ For a command that replaces a user file or replays paid work, run `create-formal ## Readiness review -An agent that did not write the code runs this review at the end of each round, and when the user asks for it. -The author of the round never writes it. The checks of the contract apply in all cases, and `just check` runs them. - -Review the extension against "What a great extension does" and against the facts of the intake. -Write the review in `readiness.md`, which starts with these three lines: - -``` -Commit: -Reviewer: -Verdict: -``` - -Then commit `readiness.md` alone. `just finish` accepts the verdict while only `readiness.md` differs from that commit. -Put the findings under `## Findings`. Give each finding its own `###` heading, or write `None.` when the review finds none. -Write each part of a finding as a bullet that starts with its bold name, such as ``- **Severity**: `warn` ``: - -- **Evidence**: the file and the line, the scenario, or the output that shows the problem -- **Command**: the command that the finding affects -- **Consequence**: what goes wrong for a user or an agent -- **Severity**: `block` when the command breaks the contract, `warn` when the adapter works around a loss, or `advise` for a better practice -- **Limit**: for a `warn` that the user accepted, the spec ID of its row under **Limits.** -- **Owner**: who can fix it, the author of the extension or the author of the script -- **Fix**: the smallest change that removes the problem - -Name a severity only in the bullet **Severity** of its finding. -`just finish` fails on a severity in a table, in fenced code, or in prose under `## Findings`. - -A loss that a user can see is `warn` or `block`, never `advise`. -Examples are data that the output keeps or changes, and a promise of `spec.md` that does not hold. -When in doubt between `warn` and `advise`, use `warn`. - -End the review with one verdict. The verdict follows from the findings, and `just finish` checks it: - -- **Ready**: no finding has the severity `block` or `warn`, and `spec.md` states no limit -- **Ready with stated limits**: no finding blocks, and each `warn` names in **Limit** a limit that the user accepted -- **Needs work**: at least one finding blocks, or a `warn` names no limit that the user accepted - -Only the user accepts a limit. Ask the user, and wait for the answer. -Then write the limit as a row under **Limits.** of its command, with the columns ID, State, Limit, and Accepted. -Its State is `limit`. Accepted records who accepted it, when, and where, such as a review comment. -A loss that you report but that the user did not accept is a `warn` finding, not a limit. - -Never reject a script because it is small, young, or private. Judge what the command does, not where the script comes from. +An agent that did not write the code runs the readiness review at the end of each round, and when the user asks for it. +Read [readiness-review.md](readiness-review.md) for the review and the form of `readiness.md`. diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md new file mode 100644 index 00000000..2566c16b --- /dev/null +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -0,0 +1,47 @@ +# Readiness review + +This file says how an agent that did not write the code reviews the extension, and how it writes `readiness.md`. + +An agent that did not write the code runs this review at the end of each round, and when the user asks for it. +The author of the round never writes it. The checks of the contract apply in all cases, and `just check` runs them. + +Review the extension against "What a great extension does" and against the facts of the intake. +Write the review in `readiness.md`, which starts with these three lines: + +``` +Commit: +Reviewer: +Verdict: +``` + +Then commit `readiness.md` alone. `just finish` accepts the verdict while only `readiness.md` differs from that commit. +Put the findings under `## Findings`. Give each finding its own `###` heading, or write `None.` when the review finds none. +Write each part of a finding as a bullet that starts with its bold name, such as ``- **Severity**: `warn` ``: + +- **Evidence**: the file and the line, the scenario, or the output that shows the problem +- **Command**: the command that the finding affects +- **Consequence**: what goes wrong for a user or an agent +- **Severity**: `block` when the command breaks the contract, `warn` when the adapter works around a loss, or `advise` for a better practice +- **Limit**: for a `warn` that the user accepted, the spec ID of its row under **Limits.** +- **Owner**: who can fix it, the author of the extension or the author of the script +- **Fix**: the smallest change that removes the problem + +Name a severity only in the bullet **Severity** of its finding. +`just finish` fails on a severity in a table, in fenced code, or in prose under `## Findings`. + +A loss that a user can see is `warn` or `block`, never `advise`. +Examples are data that the output keeps or changes, and a promise of `spec.md` that does not hold. +When in doubt between `warn` and `advise`, use `warn`. + +End the review with one verdict. The verdict follows from the findings, and `just finish` checks it: + +- **Ready**: no finding has the severity `block` or `warn`, and `spec.md` states no limit +- **Ready with stated limits**: no finding blocks, and each `warn` names in **Limit** a limit that the user accepted +- **Needs work**: at least one finding blocks, or a `warn` names no limit that the user accepted + +Only the user accepts a limit. Ask the user, and wait for the answer. +Then write the limit as a row under **Limits.** of its command, with the columns ID, State, Limit, and Accepted. +Its State is `limit`. Accepted records who accepted it, when, and where, such as a review comment. +A loss that you report but that the user did not accept is a `warn` finding, not a limit. + +Never reject a script because it is small, young, or private. Judge what the command does, not where the script comes from. From 4d711d270dbd8b3714dda007801c19bd74edd4c9 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:36:19 -0400 Subject: [PATCH 07/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20template:=20round?= =?UTF-8?q?=201=20starts=20at=20the=20intake,=20a=20build=20stops=20at=205?= =?UTF-8?q?=20rounds,=20and=20each=20case=20has=20a=20safe=20outcome?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 1 of the skill records the intake facts in vision.md, writes spec.md, commits a failing scenario at the command boundary, and only then changes the handler. AGENTS.md no longer gives another order. The skill names three acts: a pull request for review, a stop report at the limit of 5 rounds, and the delivery after just finish passes. An answer that lets the agent choose starts no round. The readiness review teaches finding IDs, states, and the ledger. A command that replaces a user file proves its invariants with the test kit, and create-formal-verification-skill stays for paid work. generic-cases.md gives the safe outcome of each case, with new cases for a large expansion, an input with several parts, a temporary file, data that a reader needs, and data that changes the meaning. acceptance.sh compares the primary output, from a reader that is not the wrapped program, with an independent reference. Part of #417. Co-Authored-By: Claude Opus 5.5 --- .../contract/extension-repository.md | 4 +- docs/north-star/contract/generic-cases.md | 45 ++++++++------ .../.agents/skills/nicely-extension/SKILL.md | 62 ++++++++++++++----- .../nicely-extension/readiness-review.md | 55 ++++++++++++++-- template/AGENTS.md.jinja | 19 +++--- template/AGENTS/rule-index.md.jinja | 16 ++++- template/acceptance.sh.jinja | 6 +- template/vision.md.jinja | 9 +++ 8 files changed, 161 insertions(+), 55 deletions(-) diff --git a/docs/north-star/contract/extension-repository.md b/docs/north-star/contract/extension-repository.md index 7d18b67f..9a4418ed 100644 --- a/docs/north-star/contract/extension-repository.md +++ b/docs/north-star/contract/extension-repository.md @@ -7,7 +7,7 @@ The repository is one Go module. Its `go.mod` pins one Nicely version, as [Compa | File | Holds | |---|---| | `README.md` | What the extension adds, and the commands that install a release | -| `vision.md` | The job of the extension in one sentence, the source and revision of each program that it wraps, and what it refuses | +| `vision.md` | The job of the extension in one sentence, the source and revision of each wrapped program, its intake facts, and what it refuses | | `spec.md` | One section per command, titled `## ncly `: what proves success, why each effect exists, each failure and its code, and the limits that the user accepted. Then `## Acceptance`: the real input, the properties that the output keeps, and each input that the wrapped program accepts but the domain refuses. Then `## Open lines`: the lines that no test proves yet. Each line of Failures and of Acceptance starts with its spec ID, as [Test kit](test-kit.md#spec-ids) says | | `manifest.toml`, `locales/`, `skill/` | The manifest, the catalogs, and the skill, which the extension embeds as [SDK](sdk.md) defines | | `worker/` | The scripts that it embeds as workers, one per command that wraps one | @@ -18,7 +18,7 @@ The repository is one Go module. Its `go.mod` pins one Nicely version, as [Compa | `AGENTS.md` | The reading order, the change steps, the recipes, and the release steps, for an agent that changes the extension | | `AGENTS/rule-index.md` | The check that enforces each rule of `AGENTS.md` | | `.agents/skills/nicely-extension/` | The skill that builds each command in rounds, from intake to a verified command, with a readiness review at the end of each round | -| `readiness.md` | The last readiness review: the commit that it reviewed, its reviewer, its verdict, and its findings | +| `readiness.md` | The last readiness review: the commit that it reviewed, its reviewer, its verdict, and its findings. Then the ledger of the build, as [Finish gate](finish-gate.md#the-ledger) defines | | `.copier-answers.yml`, `lefthook.yml` | The template version that `just update` reads, and the `just check` that Lefthook runs before each push | | `.github/workflows/check.yml` | The `just check` that GitHub runs on Ubuntu, plus macOS for public repositories, for each push to `main` and each pull request | diff --git a/docs/north-star/contract/generic-cases.md b/docs/north-star/contract/generic-cases.md index 5bfda54d..5d20df50 100644 --- a/docs/north-star/contract/generic-cases.md +++ b/docs/north-star/contract/generic-cases.md @@ -6,26 +6,33 @@ Then it adds the cases of the program that the command wraps. Each case that can occur becomes one line, with its spec ID, as [Spec IDs](spec-ids.md) defines. The line states the outcome of the case, such as a refusal with its code. +The safe outcome below is the default. Another outcome is a limit, and only the user can accept it. A case that cannot occur for the command needs no line. +An error that names the input gives its path or its item, so an agent knows which call failed. -| Group | Case | Example | -|---|---|---| -| Input | Empty | A text file of 0 bytes | -| Input | Truncated | A JSON file that ends before its last closing brace | -| Input | Unreadable | A text file without the read permission | -| Input | Of the wrong type | A file that holds a NUL byte, given to `inspect`, which counts lines of text | -| Input | Very large | A text file of several gigabytes | -| Input | With active content | A text file that makes the program load a remote file, such as a line `include https://example.com/part.txt`, or start another program, such as a line `!date` | -| Output | Exists, without `--force` | `notes.txt` exists before a command writes `notes.txt` from `notes.md` | -| Output | Is a link to the input | `notes.txt` is a symbolic link to `notes.md` | -| Output | Appears during the run | Another process writes `notes.txt` after the check and before the write | -| Output | Keeps private data of the input | `notes.md` starts with the line `author: Ana Lima`, and `notes.txt` keeps that name | -| Interruption | SIGINT or SIGTERM during the work | Ctrl-C while `inspect` counts the lines of a large file | -| Interruption | Timeout | `inspect --timeout 1ms` on a large file | -| File name | Spaces | `my notes.md` | -| File name | Leading dash | `-notes.md`, which a program can read as a flag | -| File name | Shell characters | `notes;$(date).md` | -| Other | Dry run | `inspect --dry-run` leaves the file and its folder unchanged | -| Other | Missing program | `inspect` on a host without `python3` | +| Group | Case | Example | Safe outcome | +|---|---|---|---| +| Input | Empty | A text file of 0 bytes | The result that the spec states for an empty input, or a refusal that names the input | +| Input | Truncated | A JSON file that ends before its last closing brace | A refusal of the input that is cut short or damaged, with an error that names the input. A partial output is only a limit that the user accepted | +| Input | Unreadable | A text file without the read permission | A refusal before any work, with an error that names the input | +| Input | Of the wrong type | A file that holds a NUL byte, given to `inspect`, which counts lines of text | A refusal with a code of the extension that names the input. The program never gets it | +| Input | Very large | A text file of several gigabytes | The command states a budget of work and memory. It finishes within that budget, or refuses with an error that names the input and the budget | +| Input | Large expansion | A small compressed text file that expands to many gigabytes | The command states the budget before the costly work, and refuses an input above it, with an error that names the input and the budget. The budget says if a timeout applies to each step or to all of the work | +| Input | With several parts | A text file that holds three notes, separated by lines `---` | The command keeps all parts, or refuses the input. It never keeps only the first part without a message. Only the user can accept a limit that keeps fewer parts | +| Input | With active content | A text file that makes the program load a remote file, such as a line `include https://example.com/part.txt`, or start another program, such as a line `!date` | A refusal, or a run in which the program loads no remote file and starts no program | +| Output | Exists, without `--force` | `notes.txt` exists before a command writes `notes.txt` from `notes.md` | A refusal before any work, and `notes.txt` stays unchanged | +| Output | Is a link to the input | `notes.txt` is a symbolic link to `notes.md` | A refusal, or a write that replaces the link. The command never writes through the link, and `notes.md` stays unchanged | +| Output | Appears during the run, without `--force` | Another process writes `notes.txt` after the check and before the write | The command never replaces that file. It publishes with a create that fails on an existing file, such as a hard link. When the file system cannot create the output without a replace, the command fails. It never uses a check and then a rename that replaces | +| Output | Temporary file | A link `notes.txt.tmp` to `notes.md`, made before the run | The command creates its temporary file with exclusive create, and with a name that no other process can predict. It never writes through a link at that path | +| Output | Keeps private data of the input | `notes.md` starts with the line `author: Ana Lima`, and `notes.txt` keeps that name | The command removes the private data, or the user accepts a limit | +| Output | Keeps data that a reader needs | `notes.md` starts with a header that declares its text encoding and names its author | The command keeps the data that the reader needs to show the content correctly, and removes the private text. Otherwise, the user accepts a limit | +| Output | Loses data that changes the meaning | `notes.md` declares the encoding `latin-1`, and `notes.txt` keeps its bytes without that line | The output keeps the declaration, such as a text encoding or a unit, or the command applies it to the content | +| Interruption | SIGINT or SIGTERM during the work | Ctrl-C while `inspect` counts the lines of a large file | Exit 130 or 143, with an answer that names the input. No partial output remains, the old output stays, and no child process remains | +| Interruption | Timeout | `inspect --timeout 1ms` on a large file | The timeout outcome, with an answer that names the input, even when the check of the program uses up the deadline. No partial output remains | +| File name | Spaces | `my notes.md` | The command passes the name as one path, unchanged | +| File name | Leading dash | `-notes.md`, which a program can read as a flag | The program reads it as a file, such as `./-notes.md`, never as a flag | +| File name | Shell characters | `notes;$(date).md` | The command passes the name as one path, and no shell reads it | +| Other | Dry run | `inspect --dry-run` leaves the file and its folder unchanged | No effect starts, and a refusal is the same as the refusal of a run | +| Other | Missing program | `inspect` on a host without `python3` | `PREREQ_MISSING` with exit 78 and the install hint, before any work | `inspect` is the example command of the extension template. diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index 2fc3ba82..be0dcb85 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -26,7 +26,7 @@ Get these facts before you write code. Read the source of the script, and ask it - How long it runs, what it starts in the background, and whether it can ask a question - Its exit codes, and what a repeat of the same call does -Write each fact that nobody can give as an open question in `spec.md`. +Write each fact that nobody can give as an open line of `spec.md`. ## Intake of a program @@ -34,7 +34,7 @@ A third-party program, such as an encoder, has no author to ask. Get the facts of the script intake from its documentation and its manual page. Also read its exit codes, its limits, and its known failure modes, such as the open issues of its tracker. Do not run a command that writes or bills to learn its behavior. -Write each fact that the documentation does not give as an open question in `spec.md`. +Write each fact that the documentation does not give as an open line of `spec.md`. ## Choice of operations @@ -66,23 +66,30 @@ A case that cannot occur for the command needs no line. ## Rounds +A build of a command starts at the intake and ends at "Done when" in `AGENTS.md`. +It has at most 5 rounds, and the ledger in `readiness.md` counts them. A round is new commits on the open pull request, or a new pull request. Each round ends with the independent review, the readiness review, and `just finish`. ### Round 1 -1. Implement the main path of the command, with "Change a command" in `AGENTS.md`. -2. Write every line of the command in `spec.md`. Each line that has no test stays `open`, under `## Open lines`. -3. When no line stays open, write `None.` under `## Open lines`, with a sentence that cites the generic cases. -4. Replace every `Fill:` line of `vision.md`, `spec.md`, and `qa.md`. -5. Fill in `acceptance.sh` from `## Acceptance` in `spec.md`. For a paid command, limit it to the dry run. -6. End the round. +Do these steps in this order. Commit each of the first four steps on its own: -The first implementation is incomplete by design. The open lines start the next round. +1. Record the intake facts under `## Intake` in `vision.md`, with the source and the version of each fact. The facts include the effects, the exit codes, the resource use, and the known failures. A fact that no source gives stays an open line of `spec.md`. +2. Write every line of the command in `spec.md`, from the generic cases of the contract and the intake. Each line that has no test stays `open`, under `## Open lines`. +3. Write the first scenario at the public command boundary, with its claim `# spec: `. Commit it while it fails. +4. Change the handler until that scenario passes, with "Change a command" in `AGENTS.md`. +5. When no line stays open, write `None.` under `## Open lines`, with a sentence that cites the generic cases. +6. Replace every `Fill:` line of `vision.md`, `spec.md`, and `qa.md`. +7. Fill in `acceptance.sh` from `## Acceptance` in `spec.md`. For a paid command, limit it to the dry run. +8. End the round. + +Give the commits of steps 1 to 4 to the reviewer, who records them under `## Order of round 1` in the ledger. +The first round is incomplete by design. The open lines start the next round. ### Each next round -1. Select open lines. +1. Select open lines, and the findings that the ledger records as `open`. 2. Move each line from `## Open lines` into Failures or Acceptance. 3. For each line, write a scenario with the claim `# spec: `. It fails first, for the missing behavior. 4. Change the handler until the scenario passes. The line is then `proven`. @@ -93,17 +100,42 @@ The first implementation is incomplete by design. The open lines start the next 1. Run `just check` until every check passes. 2. For a command that needs a key, relay the grant hint to the human and wait. 3. Run `just acceptance` on the real input until its independent checks pass. -4. Ask an agent that did not write the round to review the changes of the round. -5. Ask an agent that did not write the code to run the readiness review and record it in `readiness.md`. +4. Ask an agent that did not write the round to review the changes of the round. Give it the commits of the round. +5. Ask an agent that did not write the code to run the readiness review. It records the review and the ledger in `readiness.md`. One independent pass can do both reviews. 6. Write each finding that a test can prove as an open line. 7. Run `just finish`. Stop as "Done when" in `AGENTS.md` says. `just finish` fails until the loop is done. -8. Otherwise, start the next round from the open lines and the other findings. Add its commits to the open pull request, or open a new one. Base a new pull request on the branch of the previous one when that one is not merged. +8. When `just finish` fails after round 5, stop, as "Round limit" says. +9. Otherwise, start the next round from the open lines and the open findings. Add its commits to the open pull request, or open a new one. Base a new pull request on the branch of the previous one when that one is not merged. + +### Round limit + +At the limit of 5 rounds, stop. Do not start round 6. +Report to the user each open line of `spec.md`, and each finding that the ledger records as `open`, with its ID. +Then wait for the user. +An answer that lets you choose, such as "decide yourself", accepts no limit and starts no round. + +### Review, stop, and delivery + +These are three different acts: -A delivery skill or a merge gate, such as a "full stop" or "full send" request, runs after "Done when" holds. +- A pull request opens for review. It can open in round 1, before the loop is done +- A stop report goes to the user at the round limit. It delivers nothing +- The delivery, such as a merge or a release, starts only after `just finish` passes on the head + +A delivery skill or a merge gate, such as a "full stop" or "full send" request, is a delivery. It never replaces "Done when", and it never ends the rounds early. The release receipt and the human items of `qa.md` come after the merge, as "Release" in `AGENTS.md` says. -For a command that replaces a user file or replays paid work, run `create-formal-verification-skill`. +### Commands that replace a file + +For a command that replaces a user file, prove these invariants with the scenario commands of `test-kit.md` in the contract: + +- The input stays unchanged: `protect` the input before each `refuses` and `signal` call +- A failure leaves no partial output and keeps the old output: `protect` the output path, absent or old, before each `refuses` and `signal` call +- The command never writes through a link: give an output that is a link to the input, `protect` both, and refuse the call with `refuses` +- Without `--force`, the command never replaces a file that appears during the run: `appears` creates that file, and the call keeps it + +For a command that replays paid work, run `create-formal-verification-skill`. ## Readiness review diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 2566c16b..fc94932f 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -15,33 +15,76 @@ Verdict: ``` Then commit `readiness.md` alone. `just finish` accepts the verdict while only `readiness.md` differs from that commit. -Put the findings under `## Findings`. Give each finding its own `###` heading, or write `None.` when the review finds none. +`finish-gate.md` in the contract lists each check of `just finish`. + +## Findings + +Put the findings under `## Findings`, or write `None.` when no round found any. +Give each finding its own `###` heading, which starts with its ID, such as `### R2-F3 Long lines are cut`. +`R2` is the round that first reported it, and `F3` its number in that round. +Keep each finding of an earlier round, with the same ID and with a state for this round. Write each part of a finding as a bullet that starts with its bold name, such as ``- **Severity**: `warn` ``: - **Evidence**: the file and the line, the scenario, or the output that shows the problem - **Command**: the command that the finding affects - **Consequence**: what goes wrong for a user or an agent - **Severity**: `block` when the command breaks the contract, `warn` when the adapter works around a loss, or `advise` for a better practice -- **Limit**: for a `warn` that the user accepted, the spec ID of its row under **Limits.** +- **State**: `open`, `fixed`, `limit`, or `declined`. A finding of this round can have no state yet +- **Spec**: the spec ID of its line: the open line, the line that the test of the fix proves, or the accepted limit +- **Commit**: for a `fixed` finding, the commit that fixed it +- **Reason**: for a `declined` finding, the reason of the author - **Owner**: who can fix it, the author of the extension or the author of the script - **Fix**: the smallest change that removes the problem Name a severity only in the bullet **Severity** of its finding. -`just finish` fails on a severity in a table, in fenced code, or in prose under `## Findings`. +After the first three lines, `just finish` fails on the words block, warn, or advise anywhere else. +Examples are a table, fenced code, and prose. A loss that a user can see is `warn` or `block`, never `advise`. Examples are data that the output keeps or changes, and a promise of `spec.md` that does not hold. When in doubt between `warn` and `advise`, use `warn`. +Never lower the severity of a finding in a later round. + +A `warn` or `block` finding closes only as `fixed` or `limit`. Only an `advise` can be `declined`. +A change of `spec.md` that narrows a promise to close a `warn` or `block` finding is a `limit`. Only the user can accept it. +`None.` under `## Open lines` does not close a finding that the ledger records as `open`. End the review with one verdict. The verdict follows from the findings, and `just finish` checks it: -- **Ready**: no finding has the severity `block` or `warn`, and `spec.md` states no limit -- **Ready with stated limits**: no finding blocks, and each `warn` names in **Limit** a limit that the user accepted -- **Needs work**: at least one finding blocks, or a `warn` names no limit that the user accepted +- **Ready**: each `block` or `warn` finding is `fixed`, and `spec.md` states no limit +- **Ready with stated limits**: each `block` finding is `fixed`, and each `warn` is `fixed` or a `limit` that the user accepted +- **Needs work**: any other case Only the user accepts a limit. Ask the user, and wait for the answer. Then write the limit as a row under **Limits.** of its command, with the columns ID, State, Limit, and Accepted. Its State is `limit`. Accepted records who accepted it, when, and where, such as a review comment. A loss that you report but that the user did not accept is a `warn` finding, not a limit. +## Ledger + +After the findings, keep the ledger of the build. Add one row for this round, and keep the rows of earlier rounds: + +``` +## Rounds + +| Round | Head | Change review | Change commits | Readiness review | Finish | +|---|---|---|---|---|---| +| 1 | | | .. | | `fail` | +| 2 | | | .. | | `pending` | + +## Order of round 1 + +| Step | Commit | +|---|---| +| Intake | | +| Spec | | +| Scenario | | +| Handler | | +``` + +Change commits names the commits that the change review covered. The readiness review covers Head. +When one pass did both reviews, name the same reviewer in both cells. +Finish holds the result of `just finish` for each earlier round, and `pending` for this round. +A build has at most 5 rounds. + Never reject a script because it is small, young, or private. Judge what the command does, not where the script comes from. diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index 29496f38..6b347786 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -17,20 +17,19 @@ Read `extension-identity.md` in the pinned contract for the slug and command own ## Change a command Build each command with the skill in `.agents/skills/nicely-extension/`, from the first command. -It runs these steps in rounds: - -1. Write the command's spec before its implementation. -2. Prepare the manifest, catalog, and placeholder handler until `extension.Check` passes. -3. Write a scenario that fails for the missing behavior. -4. Write the handler body, on `sdk/wrap` for one program, or on `sdk/runner` for more. -5. Teach when to use the command in `skill/SKILL.md`. +Its rounds give the order of the work, from the intake to "Done when". +Write the handler body on `sdk/wrap` for one program, or on `sdk/runner` for more. +Declare the command in the manifest and the catalog, so `extension.Check` passes. +Teach when to use the command in `skill/SKILL.md`. Give each command one bounded job beneath the extension root. Never relay arbitrary arguments to a wrapped program. The skill teaches when to use the command, rather than its implementation. For a command with a dry run, tell the agent to dry-run the call first. -In `acceptance.sh`, judge the result with a tool other than the extension. +In `acceptance.sh`, judge the primary output with a reader that is not the wrapped program. +A check that reads only a header or the file type is not sufficient. +Compare the content that this reader gives with an independent reference. Use the real input and criteria of `## Acceptance`. For a paid command, run only its dry run. @@ -90,7 +89,9 @@ The rounds of the skill stop when all of these hold: - `just finish` passes - `just check` and `just acceptance` pass, and the hosted checks pass -After 5 rounds, stop anyway, and report each open line to the user. +A build has at most 5 rounds, and `just finish` fails after 5. +At that limit, stop, and report each open line and each open finding to the user, as the skill says. +An answer that lets you choose, such as "decide yourself", accepts no limit and starts no round. For a command that needs a key, relay the grant hint to the human and wait. A delivery skill or a merge gate, such as a "full stop" or "full send" request, runs after "Done when" holds. It never replaces "Done when". `just signoff` runs `just finish` first, so a merge or a release waits until `just finish` passes. When the previous pull request is not merged, a new pull request stacks on its branch. diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index dbdd77f8..91889fd1 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -17,15 +17,21 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | The skill names only declared commands | [Kit skill links][links] | | Sign off through `just signoff` and release through `just release vX.Y.Z` | [Signoff receipt][signoff], [release receipt][release] | | A copied recipe holds no script | [Core template test][template-test] | -| `acceptance.sh` judges with another tool | Prose only; [release runs acceptance][release] | +| `acceptance.sh` compares the primary output, from a reader that is not the wrapped program, with an independent reference | Prose only; [release runs acceptance][release] | | A test claims each line of Failures and Acceptance in `spec.md` | [Kit spec IDs][spec] | | One bounded job, no arbitrary argument relay; the skill teaches when | Prose only | -| Spec first; a scenario fails before the handler body exists | Prose only | +| Round 1 records the intake, then the spec, then a failing scenario, then the handler | [Finish gate][finish] checks their commits in the [ledger][ledger] | | Build each command with the `nicely-extension` skill, in rounds from its first pull request | Prose only | | The first round writes every line of `spec.md`; a line without a test stays `open` | Prose only | | Each round ends with an independent review and the readiness review | [Signoff][signoff] runs the [finish gate][finish], which needs a readiness review of the signed commit by a named reviewer | | At the exit, filled forms, no open line, accepted limits, and a passing readiness verdict for this commit, by a named reviewer, from its findings | [Finish gate][finish] | -| Stop at "Done when" or after 5 rounds, and deliver only after that | [Signoff][signoff] runs the [finish gate][finish], and [release][release] needs that signoff | +| A build has at most 5 rounds | The [ledger][ledger] counts them, and the [finish gate][finish] fails after 5 | +| At the limit, stop and report; an answer that lets the agent choose starts no round | Prose only | +| A pull request for review, a stop report, and the delivery are three acts; deliver only after "Done when" | [Signoff][signoff] runs the [finish gate][finish], and [release][release] needs a receipt with `just finish` | +| The ledger names the head, the reviewers, the commits that each review covered, and the result of `just finish` of each round | [Ledger][ledger] | +| Each finding keeps its ID and its highest severity, and gets a state each round; a `warn` or `block` closes only as `fixed` or `limit` | [Ledger findings][ledger-findings] | +| A change of `spec.md` that narrows a promise to close a `warn` or `block` is a limit that only the user accepts | Prose only | +| A command that replaces a user file proves its invariants with `protect`, `refuses`, `signal`, and `appears` | [Kit `refuses`][refuses], [kit `signal`][signal], [kit `appears`][appears] | [calls]: https://github.com/pascalandy/nicely/blob/main/sdk/extensiontest/calls.go [links]: https://github.com/pascalandy/nicely/blob/main/sdk/extensiontest/links.go @@ -34,8 +40,12 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". [codes]: https://github.com/pascalandy/nicely/blob/main/sdk/extensiontest/codes.go [refuses]: https://github.com/pascalandy/nicely/blob/main/sdk/extensiontest/paired.go [signal]: https://github.com/pascalandy/nicely/blob/main/sdk/extensiontest/signal.go +[appears]: https://github.com/pascalandy/nicely/blob/main/sdk/extensiontest/appears.go [spec]: https://github.com/pascalandy/nicely/blob/main/sdk/extensiontest/spec.go + [finish]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/finish.go +[ledger]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger.go +[ledger-findings]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger_findings.go [signoff]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/signoff.go [release]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/release.go [template-test]: https://github.com/pascalandy/nicely/blob/main/scripts/template-test.sh diff --git a/template/acceptance.sh.jinja b/template/acceptance.sh.jinja index 4ae0f591..4265b517 100644 --- a/template/acceptance.sh.jinja +++ b/template/acceptance.sh.jinja @@ -69,7 +69,11 @@ fi [[ "$status" == 0 ]] || die "ncly ${command[*]} exited $status" grep -qF '"ok":true' "$scratch/answer" || die 'the answer holds no "ok":true' -# Fill: check each property of ## Acceptance with a tool other than the extension. +# Fill: check each property of ## Acceptance on the primary output, with a reader +# that is not the wrapped program. Compare the content that this reader gives +# with an independent reference. A check of only a header or of the file type is +# not sufficient. Here, wc and awk read the input again, and the answer must +# match their counts. # spec: size grep -qE "\"size_bytes\":$(wc -c <"$input" | tr -d ' ')[,}]" "$scratch/answer" || die "the size differs from wc -c" # spec: lines diff --git a/template/vision.md.jinja b/template/vision.md.jinja index e799ece8..e8950b84 100644 --- a/template/vision.md.jinja +++ b/template/vision.md.jinja @@ -14,6 +14,15 @@ Fill: the source and the revision of each program or script that a command wraps - `worker/inspect.py`, which the extension template wrote +## Intake + +Fill: each fact of the intake of each wrapped program, with its source and the version of that source. Give its effects, its exit codes, its resource use, and its known failures. A fact that no source gives is an open line of `spec.md`. + +- Effects: `worker/inspect.py` reads the file once, in chunks of 64 KiB, and writes nothing. Source: `worker/inspect.py` of the template, at the Nicely version that `go.mod` pins +- Exit codes: 0 with one JSON object on stdout, and 2 for a wrong argument count. Source: the same file and version +- Resource use: memory stays near one chunk, whatever the size of the file. Source: the same file and version +- Known failures: Python raises an error and exits 1 on a file that it cannot open. Source: the same file and version + ## Refuses Fill: what the extension will not do, and why. From 62cbebe9d0bc8db15f3a7f8d671179c563c960ac Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:38:10 -0400 Subject: [PATCH 08/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20finish?= =?UTF-8?q?=20reads=20the=20history=20of=20a=20build=20from=20its=20intake?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The history of readiness.md went back before the build. So the build of a second command had to keep the findings of the first, and a new finding could reuse an old ID. Finish now reads only the reviews committed after the Intake commit of the build. Part of #417. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 2 +- scripts/template-finish-helpers.sh | 13 +++++++------ scripts/template-ledger-test.sh | 11 +++++++++++ sdk/extensionrepo/ledger.go | 8 +++++++- sdk/extensionrepo/ledger_findings.go | 17 +++++++++++------ 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index 99bdc573..2f65d5c6 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -54,6 +54,6 @@ A finding of the last round can have no state yet. - A finding without an ID, a finding of an earlier round without a state, or a state without its parts - A finding with the state `open`, even when `## Open lines` holds `None.` - A `warn` or `block` finding with the state `declined` -- A finding of an earlier committed `readiness.md` that is missing, or whose severity went down +- A finding of an earlier committed `readiness.md` that is missing, or whose severity went down. Finish reads the commits after the Intake commit of the build, so an earlier build is no history of this one At 5 rounds, a failing `just finish` also tells the agent to stop and to report each open line and each open finding. diff --git a/scripts/template-finish-helpers.sh b/scripts/template-finish-helpers.sh index 2dea96bc..004033c7 100644 --- a/scripts/template-finish-helpers.sh +++ b/scripts/template-finish-helpers.sh @@ -26,7 +26,7 @@ fct_finish_said() { # fct_readiness writes readiness.md for the commit $1, with the verdict $2, the # findings $3 under ## Findings, and the reviewer $4. An empty $3 leaves out # ## Findings, and an empty $4 leaves out the line Reviewer:. The ledger -# holds $5 rounds, or 1, and its last round reviewed $1. +# holds $5 rounds, or 1, from the intake $6, and its last round reviewed $1. fct_readiness() { local reviewer="${4-an agent that wrote none of the code}" { @@ -34,16 +34,17 @@ fct_readiness() { [[ -z "${reviewer}" ]] || printf 'Reviewer: %s\n' "${reviewer}" printf 'Verdict: %s\n\n# Readiness review\n' "${2}" [[ -z "${3}" ]] || printf '\n## Findings\n\n%s\n' "${3}" - fct_ledger "${finish}" "${1}" "${5:-1}" + fct_ledger "${finish}" "${1}" "${5:-1}" "${6:-}" } >"${finish}/readiness.md" } # fct_ledger prints the ledger of a build of $3 rounds in the repository $1, -# whose last round reviewed $2. Earlier rounds name the first commit, and -# round 1 commits its handler in $2 (#417). +# whose last round reviewed $2. The build starts at the intake $4, or at the +# first commit. Earlier rounds name that intake, and round 1 commits its +# handler in $2 (#417). fct_ledger() { - local first i - first="$(git -C "${1}" rev-list --max-parents=0 HEAD)" + local first="${4}" i + [[ -n "${first}" ]] || first="$(git -C "${1}" rev-list --max-parents=0 HEAD)" printf '\n## Rounds\n\n| Round | Head | Change review | Change commits | Readiness review | Finish |\n|---|---|---|---|---|---|\n' for ((i = 1; i < ${3}; i++)); do printf '| %s | %s | agent A | %s..%s | agent B | `fail` |\n' "${i}" "${first}" "${first}" "${first}" diff --git a/scripts/template-ledger-test.sh b/scripts/template-ledger-test.sh index a0cfef0b..ebeb78af 100644 --- a/scripts/template-ledger-test.sh +++ b/scripts/template-ledger-test.sh @@ -105,4 +105,15 @@ fct_test_ledger() { printf '\357\273\277' | cat - "${finish}/readiness.md" >"${scratch}/readiness.md" mv "${scratch}/readiness.md" "${finish}/readiness.md" fct_finish 0 + + # A new build, such as of another command, starts at its own intake. + # The findings of the build before it are no history of the new build. + git -C "${finish}" checkout -q readiness.md + printf 'Intake of the next build.\n' >>"${finish}/README.md" + git -C "${finish}" commit -qam 'intake of the next build' + first="$(git -C "${finish}" rev-parse HEAD)" + printf 'Handler of the next build.\n' >>"${finish}/README.md" + git -C "${finish}" commit -qam 'handler of the next build' + fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready "${advise}${declined}" 'agent B' 1 "${first}" + fct_finish 0 } diff --git a/sdk/extensionrepo/ledger.go b/sdk/extensionrepo/ledger.go index 27b78b39..1a70a70d 100644 --- a/sdk/extensionrepo/ledger.go +++ b/sdk/extensionrepo/ledger.go @@ -66,7 +66,13 @@ func ledgerProblems(readiness, reviewed string, findings []finding, spec string, } problems = append(problems, orderProblems(order)...) problems = append(problems, stateProblems(findings, len(rounds), spec, limits)...) - return append(problems, historyProblems(findings)...) + // The build starts at its intake, so the reviews of an earlier build, + // such as of another command, are no history of this one. + since := "" + if intake, found := order["Intake"]; found { + since, _ = ledgerCommit(intake.cells[1]) + } + return append(problems, historyProblems(findings, since)...) } // roundProblems checks one row of ## Rounds, the round number of its place. diff --git a/sdk/extensionrepo/ledger_findings.go b/sdk/extensionrepo/ledger_findings.go index d7e50049..7900d102 100644 --- a/sdk/extensionrepo/ledger_findings.go +++ b/sdk/extensionrepo/ledger_findings.go @@ -66,11 +66,16 @@ func stateProblems(findings []finding, n int, spec string, limits []limit) []str return problems } -// historyProblems compares the findings with each committed readiness.md of -// this branch. A finding keeps its ID in each later review, and its severity -// never goes down, so no loss leaves the ledger. -func historyProblems(findings []finding) []string { - revisions, err := exec.Command("git", "log", "--format=%H", "HEAD", "--", "readiness.md").Output() +// historyProblems compares the findings with each readiness.md committed +// after since, the intake of the build, or in all of HEAD without it. A +// finding keeps its ID in each later review, and its severity never goes +// down, so no loss leaves the ledger. +func historyProblems(findings []finding, since string) []string { + revisions := "HEAD" + if since != "" { + revisions = since + "..HEAD" + } + log, err := exec.Command("git", "log", "--format=%H", revisions, "--", "readiness.md").Output() if err != nil { return []string{"readiness.md: git log: " + err.Error()} } @@ -80,7 +85,7 @@ func historyProblems(findings []finding) []string { } var problems []string reported := map[string]bool{} - for _, revision := range strings.Fields(string(revisions)) { + for _, revision := range strings.Fields(string(log)) { old, err := exec.Command("git", "show", revision+":readiness.md").Output() if err != nil { continue From a1c2fdb13925ef09b7c754c3745baee45c89fe73 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:24:31 -0400 Subject: [PATCH 09/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20sdk/wrap?= =?UTF-8?q?:=20move=20the=20program=20check=20into=20check.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The check moves without a change of behavior, so that wrap.go keeps room under the size limit for the next fix. Co-Authored-By: Claude Opus 5.5 --- sdk/wrap/check.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ sdk/wrap/wrap.go | 35 ----------------------------------- 2 files changed, 47 insertions(+), 35 deletions(-) create mode 100644 sdk/wrap/check.go diff --git a/sdk/wrap/check.go b/sdk/wrap/check.go new file mode 100644 index 00000000..bcf6a01a --- /dev/null +++ b/sdk/wrap/check.go @@ -0,0 +1,47 @@ +package wrap + +import ( + "cmp" + "context" + "strings" + "time" + + "github.com/pascalandy/nicely/sdk/contract" + "github.com/pascalandy/nicely/sdk/i18n" + "github.com/pascalandy/nicely/sdk/runner" +) + +// check checks the program with the rule of Required programs in +// core-spec.md. It returns where PATH found it, or the outcome that stops +// the call. That outcome is the one of a signal, or PREREQ_MISSING, whose +// hint and next hold the install command of this machine. +func check(ctx context.Context, printer *i18n.Printer, name string, r contract.Requirement, env []string, limit time.Duration) (string, *contract.Outcome) { + found, err := runner.CheckProgram(ctx, name, r, env, limit) + data := map[string]any{"Name": name, "Found": found.Version, "Version": r.Version, "Exit": found.Exit, "Probe": strings.Join(append([]string{name}, r.ProbeArgs()...), " ")} + var message, hint string + switch { + case err != nil: + data["Exit"] = -1 + message = printer.T("wrap.probe_failed", data) + case found.Signal != nil: + return "", &contract.Outcome{Signal: contract.StopSignal(found.Signal)} + case found.Verdict == runner.ProgramFound: + return found.Path, nil + case found.Verdict == runner.ProgramMissing: + message = printer.T("wrap.missing", data) + case found.Verdict == runner.ProgramTooOld: + message = printer.T("wrap.too_old", data) + case found.Verdict == runner.ProgramProbeFailed: + message = printer.T("wrap.probe_failed", data) + case found.Verdict == runner.ProgramNoVersion: + message = printer.T("wrap.no_version", data) + default: + // ProgramCheck.Failed leaves a probe that timed out to its caller. + message, hint = printer.T("wrap.probe_timeout", data), printer.T("wrap.probe_timeout_hint", data) + } + problem := contract.Problem{Code: contract.PrereqMissing, Message: message, Hint: cmp.Or(hint, printer.T("wrap.install", data))} + if found.Install != nil { + problem.Hint, problem.Next = strings.Join(found.Install, " "), found.Install + } + return "", &contract.Outcome{Errors: []contract.Problem{problem}} +} diff --git a/sdk/wrap/wrap.go b/sdk/wrap/wrap.go index 6ef6539b..e36a2b97 100644 --- a/sdk/wrap/wrap.go +++ b/sdk/wrap/wrap.go @@ -149,41 +149,6 @@ func Run(ctx context.Context, call extension.Call, p Program, decode func(Ended) return decode(Ended{Exit: result.Exit, Stdout: stdout.Bytes(), Stderr: stderr.Bytes(), name: p.Name, printer: printer}) } -// check checks the program with the rule of Required programs in -// core-spec.md. It returns where PATH found it, or the outcome that stops -// the call. That outcome is the one of a signal, or PREREQ_MISSING, whose -// hint and next hold the install command of this machine. -func check(ctx context.Context, printer *i18n.Printer, name string, r contract.Requirement, env []string, limit time.Duration) (string, *contract.Outcome) { - found, err := runner.CheckProgram(ctx, name, r, env, limit) - data := map[string]any{"Name": name, "Found": found.Version, "Version": r.Version, "Exit": found.Exit, "Probe": strings.Join(append([]string{name}, r.ProbeArgs()...), " ")} - var message, hint string - switch { - case err != nil: - data["Exit"] = -1 - message = printer.T("wrap.probe_failed", data) - case found.Signal != nil: - return "", &contract.Outcome{Signal: contract.StopSignal(found.Signal)} - case found.Verdict == runner.ProgramFound: - return found.Path, nil - case found.Verdict == runner.ProgramMissing: - message = printer.T("wrap.missing", data) - case found.Verdict == runner.ProgramTooOld: - message = printer.T("wrap.too_old", data) - case found.Verdict == runner.ProgramProbeFailed: - message = printer.T("wrap.probe_failed", data) - case found.Verdict == runner.ProgramNoVersion: - message = printer.T("wrap.no_version", data) - default: - // ProgramCheck.Failed leaves a probe that timed out to its caller. - message, hint = printer.T("wrap.probe_timeout", data), printer.T("wrap.probe_timeout_hint", data) - } - problem := contract.Problem{Code: contract.PrereqMissing, Message: message, Hint: cmp.Or(hint, printer.T("wrap.install", data))} - if found.Install != nil { - problem.Hint, problem.Next = strings.Join(found.Install, " "), found.Install - } - return "", &contract.Outcome{Errors: []contract.Problem{problem}} -} - // runtime returns the RUNTIME outcome of a message, whose hint warns that // a repeat is not safe, because the program may have made its effects. func runtime(message string, printer *i18n.Printer) contract.Outcome { From c4bc0f7ff2679271748a429235c87797145211de Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:34:18 -0400 Subject: [PATCH 10/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20testdata?= =?UTF-8?q?:=20move=20the=20stop=20cases=20of=20the=20hello=20wrap=20scena?= =?UTF-8?q?rio=20to=20wrap=5Fstop.txtar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The timeout, Ctrl-C, and version probe cases move without a change of behavior, so that wrap.txtar keeps room under the size limit. Co-Authored-By: Claude Opus 5.5 --- docs/milestones/M18-extension-template.md | 2 +- .../hello/testdata/script/wrap.txtar | 65 +------------- .../hello/testdata/script/wrap_stop.txtar | 84 +++++++++++++++++++ 3 files changed, 87 insertions(+), 64 deletions(-) create mode 100644 testdata/extensions/hello/testdata/script/wrap_stop.txtar diff --git a/docs/milestones/M18-extension-template.md b/docs/milestones/M18-extension-template.md index 8ef39916..4aab6870 100644 --- a/docs/milestones/M18-extension-template.md +++ b/docs/milestones/M18-extension-template.md @@ -57,7 +57,7 @@ Once this card ships, remove from the docs each marker that names M18-T3, such a ### M18-T4 Wrap programs and embed workers through the SDK - **Read:** [Wrapped programs](../north-star/contract/wrap.md), [Programs that ncly runs](../north-star/contract/programs-that-ncly-runs.md), [Commands that core calls](../north-star/contract/commands-that-core-calls.md), [Required programs](../north-star/core-spec.md#required-programs), [SDK](../north-star/contract/sdk.md), [D047](../north-star/decisions/D047-terminal-process-group.md) -- **Proves:** `sdk/wrap/`, `testdata/extensions/hello/testdata/script/wrap.txtar`, and `testdata/script/session.txtar`. They cover a missing program, a version probe that times out, an exit other than 0, malformed output, and a timeout with exit 1. They also cover Ctrl-C with exit 130, an empty stdin, and a child environment with only the declared keys. A worker reads ./input.md from the caller's folder, and an open of /dev/tty fails at once under a real PTY, on Linux and macOS +- **Proves:** `sdk/wrap/`, `testdata/extensions/hello/testdata/script/wrap.txtar`, `wrap_stop.txtar` beside it, and `testdata/script/session.txtar`. They cover a missing program, a version probe that times out, an exit other than 0, malformed output, and a timeout with exit 1. They also cover Ctrl-C with exit 130, an empty stdin, and a child environment with only the declared keys. A worker reads ./input.md from the caller's folder, and an open of /dev/tty fails at once under a real PTY, on Linux and macOS Add `sdk/wrap`, as [Wrapped programs](../north-star/contract/wrap.md) defines, and state its default size limit there. Add `Resources` to `extension.Extension`, with the extraction that [SDK](../north-star/contract/sdk.md) defines. Give the `hello` fixture one wrapped command with a worker, which the scenarios run. diff --git a/testdata/extensions/hello/testdata/script/wrap.txtar b/testdata/extensions/hello/testdata/script/wrap.txtar index 7ec8c732..31e42b87 100644 --- a/testdata/extensions/hello/testdata/script/wrap.txtar +++ b/testdata/extensions/hello/testdata/script/wrap.txtar @@ -1,5 +1,6 @@ # The Go fixture's hello count runs its worker count.sh with bash, its declared program, -# through sdk/wrap, as Wrapped programs defines. hello scan runs scan.sh the same way, but +# through sdk/wrap, as Wrapped programs defines. wrap_stop.txtar covers the timeout, the +# signals, and the version probe. hello scan runs scan.sh the same way, but # discards its stdout, which holds only progress. keyed relay passes a key to the program # keytool the same way. stub/bash stands in for bash: it answers the version probe, writes the # path of the worker and the mode of its folder, then does what STUB says, by default running @@ -89,51 +90,6 @@ exits 1 ncly hello scan input.md --json answer stderr RUNTIME stderr '"message":"bash failed with exit 3\.' -# At the timeout, the whole process tree stops, a child in its own process group included, -# with exit 1, and the folder of the worker goes -env STUB=hang -exits 1 ncly hello count input.md --timeout 1s --json -answer stderr RUNTIME -stderr '"message":"bash did not finish within 1 s, so it was stopped\.' -gone hang.pid -gone child.pid -exec /bin/sh -c 'test ! -e "$(cat worker.path)"' - -# Ctrl-C stops the process tree too, with exit 130, and the folder of the worker goes -rm hang.pid child.pid worker.path -exec exitcode exit.txt ncly hello count input.md --json & -waitfile hang.pid -kill -INT -wait -grep '^130$' exit.txt -! stdout . -answer stderr INTERRUPTED -gone hang.pid -gone child.pid -exec /bin/sh -c 'test ! -e "$(cat worker.path)"' - -# A program that ignores Ctrl-C runs until the SIGKILL of core's cleanup, 4 seconds in the -# test binary, and the folder of the worker is gone before that SIGKILL -rm hang.pid child.pid worker.path -env STUB=stubborn -exec exitcode exit.txt ncly hello count input.md --json & -waitfile stubborn.pid -kill -INT -wait -grep '^130$' exit.txt -answer stderr INTERRUPTED -gone stubborn.pid -exec /bin/sh -c 'test ! -e "$(cat worker.path)"' - -# A version probe that does not finish in time is PREREQ_MISSING with exit 78, before work -rm worker.path -env STUB=probe-hang -exits 78 ncly hello count input.md --timeout 1s --json -answer stderr PREREQ_MISSING -stderr '"message":"bash --version did not print its version in time\.","hint":"Check that bash --version prints a version and exits\."\}' -gone probe.pid -! exists worker.path - # The program gets the variables of its caller, except each key that its command does not # pass: hello count passes none, even run directly with a key in its environment env STUB=env @@ -193,10 +149,6 @@ DEEPGRAM_API_KEY c80d260f9714b7fa16564b76cb5fb1359ccc0a6284eb5cc4b152b3fdd6290ba -- stub/bash -- #!/bin/sh if [ "$1" = --version ]; then - if [ "$STUB" = probe-hang ]; then - echo $$ > "$WORK/probe.pid" - exec /bin/sleep 30 - fi echo 'GNU bash, version 5.2.0(1)-release' exit 0 fi @@ -223,19 +175,6 @@ noisy) progress) /usr/bin/yes 'Encoding: task 1 of 1, 50.00 %' | /usr/bin/head -c 1100000 ;; -hang) - pgroup /bin/sh -c 'echo $$ > "$WORK/child.tmp"; mv "$WORK/child.tmp" "$WORK/child.pid"; exec /bin/sleep 30' >/dev/null 2>&1 & - while [ ! -f "$WORK/child.pid" ]; do /bin/sleep 0.01; done - echo $$ > "$WORK/hang.tmp" - mv "$WORK/hang.tmp" "$WORK/hang.pid" - exec /bin/sleep 30 - ;; -stubborn) - trap '' INT TERM - echo $$ > "$WORK/stubborn.tmp" - mv "$WORK/stubborn.tmp" "$WORK/stubborn.pid" - exec /bin/sleep 30 - ;; env) { /usr/bin/env | /usr/bin/sed -n 's/^\([A-Za-z0-9_]*_API_KEY\)=.*/key \1/p'; echo "HELLO_PLAIN=$HELLO_PLAIN"; } > "$WORK/env.log" echo '{"lines":0}' diff --git a/testdata/extensions/hello/testdata/script/wrap_stop.txtar b/testdata/extensions/hello/testdata/script/wrap_stop.txtar new file mode 100644 index 00000000..78fdf3bc --- /dev/null +++ b/testdata/extensions/hello/testdata/script/wrap_stop.txtar @@ -0,0 +1,84 @@ +# hello count runs its worker count.sh with bash through sdk/wrap, as wrap.txtar shows. This +# scenario covers what stops the program: the timeout, Ctrl-C, and a version probe that does +# not finish. stub/bash stands in for bash. It answers the version probe, writes the path of +# the worker, then does what STUB says. +chmod 755 stub/bash +env PATH=$WORK/stub:$PATH +env LANG= LC_ALL= LC_MESSAGES= NCLY_LANG= + +# At the timeout, the whole process tree stops, a child in its own process group included, +# with exit 1, and the folder of the worker goes +env STUB=hang +exits 1 ncly hello count input.md --timeout 1s --json +answer stderr RUNTIME +stderr '"message":"bash did not finish within 1 s, so it was stopped\.' +gone hang.pid +gone child.pid +exec /bin/sh -c 'test ! -e "$(cat worker.path)"' + +# Ctrl-C stops the process tree too, with exit 130, and the folder of the worker goes +rm hang.pid child.pid worker.path +exec exitcode exit.txt ncly hello count input.md --json & +waitfile hang.pid +kill -INT +wait +grep '^130$' exit.txt +! stdout . +answer stderr INTERRUPTED +gone hang.pid +gone child.pid +exec /bin/sh -c 'test ! -e "$(cat worker.path)"' + +# A program that ignores Ctrl-C runs until the SIGKILL of core's cleanup, 4 seconds in the +# test binary, and the folder of the worker is gone before that SIGKILL +rm hang.pid child.pid worker.path +env STUB=stubborn +exec exitcode exit.txt ncly hello count input.md --json & +waitfile stubborn.pid +kill -INT +wait +grep '^130$' exit.txt +answer stderr INTERRUPTED +gone stubborn.pid +exec /bin/sh -c 'test ! -e "$(cat worker.path)"' + +# A version probe that does not finish in time is PREREQ_MISSING with exit 78, before work +rm worker.path +env STUB=probe-hang +exits 78 ncly hello count input.md --timeout 1s --json +answer stderr PREREQ_MISSING +stderr '"message":"bash --version did not print its version in time\.","hint":"Check that bash --version prints a version and exits\."\}' +gone probe.pid +! exists worker.path + +-- input.md -- +One line +-- stub/bash -- +#!/bin/sh +if [ "$1" = --version ]; then + if [ "$STUB" = probe-hang ]; then + echo $$ > "$WORK/probe.pid" + exec /bin/sleep 30 + fi + echo 'GNU bash, version 5.2.0(1)-release' + exit 0 +fi +echo "$1" > "$WORK/worker.path" +case "$STUB" in +hang) + pgroup /bin/sh -c 'echo $$ > "$WORK/child.tmp"; mv "$WORK/child.tmp" "$WORK/child.pid"; exec /bin/sleep 30' >/dev/null 2>&1 & + while [ ! -f "$WORK/child.pid" ]; do /bin/sleep 0.01; done + echo $$ > "$WORK/hang.tmp" + mv "$WORK/hang.tmp" "$WORK/hang.pid" + exec /bin/sleep 30 + ;; +stubborn) + trap '' INT TERM + echo $$ > "$WORK/stubborn.tmp" + mv "$WORK/stubborn.tmp" "$WORK/stubborn.pid" + exec /bin/sleep 30 + ;; +*) + exec /bin/bash "$@" + ;; +esac From 17b02ba797ce4a5816b53de8ae3166b59b4bc354 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:35:34 -0400 Subject: [PATCH 11/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20sdk/wrap:=20name=20?= =?UTF-8?q?the=20input=20in=20signal=20and=20timeout=20answers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Program.Input holds the item or path that the call works on. When it is set, each RUNTIME message that Run and Ended make names it. A signal adds an INTERRUPTED or TERMINATED error that names it, which Finish puts first. An empty Input keeps the messages of today. A version probe that uses up the deadline of Timeout now gives the timeout, RUNTIME with exit 1, instead of PREREQ_MISSING. A probe that reaches ProbeLimit inside a longer timeout stays PREREQ_MISSING. hello count sets its file as the input, and hello scan sets none, so the wrap scenarios prove both paths. Refs #407 Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/wrap.md | 8 +- sdk/i18n/locales/en.toml | 30 ++++++- sdk/wrap/check.go | 17 ++-- sdk/wrap/wrap.go | 87 +++++++++++++++---- testdata/extensions/hello/count.go | 5 +- .../hello/testdata/script/wrap.txtar | 18 ++-- .../hello/testdata/script/wrap_stop.txtar | 52 +++++++++-- 7 files changed, 171 insertions(+), 46 deletions(-) diff --git a/docs/north-star/contract/wrap.md b/docs/north-star/contract/wrap.md index 21753c70..72251799 100644 --- a/docs/north-star/contract/wrap.md +++ b/docs/north-star/contract/wrap.md @@ -14,12 +14,14 @@ A wrapped program runs through the program runner of [Programs that ncly runs](p | Case | Result | |---|---| -| The program is missing or too old, or its version probe fails or times out | `PREREQ_MISSING` before any work, as [Required programs](../core-spec.md#required-programs) defines. The probe gets at most 10 seconds, within the command's timeout. `ProgramCheck.Failed` leaves a timeout to its caller, so `sdk/wrap` maps it | +| The program is missing or too old, or its version probe fails or does not finish within 10 seconds | `PREREQ_MISSING` before any work, as [Required programs](../core-spec.md#required-programs) defines. The probe gets at most 10 seconds, within the command's timeout. `ProgramCheck.Failed` leaves a timeout to its caller, so `sdk/wrap` maps it | | The input is invalid | `USAGE_INVALID`, which the handler answers before it calls `sdk/wrap` | | The program exits with a code other than 0, or writes on stderr | The outcome that the handler maps for that exit, and `RUNTIME` for any other exit. No exit code of the program passes through unmapped, and its own 75 never makes a repeat safe | | The output is missing, malformed, or over the size limit | `RUNTIME`. A decoder never turns a failure into a success. Only stdout counts for the limit, and stderr keeps its first bytes. Discarded stdout never counts | -| The command's timeout expires | The process tree stops, and `sdk/wrap` answers `RUNTIME` with exit 1. Exit 75 needs [Retry safety](retry-safety.md), which only the handler can show, so such a handler uses `sdk/runner` | -| Ctrl-C or SIGTERM arrives | The runner signals the process tree, and the exit is 130 or 143 | +| The command's timeout expires, also while the version probe runs | The process tree stops, and `sdk/wrap` answers `RUNTIME` with exit 1. Exit 75 needs [Retry safety](retry-safety.md), which only the handler can show, so such a handler uses `sdk/runner` | +| Ctrl-C or SIGTERM arrives | The runner signals the process tree, and the exit is 130 or 143. With an input, the leading error, `INTERRUPTED` or `TERMINATED`, names it | + +The handler can give `sdk/wrap` its input, the item or path that the call works on. The message of each `RUNTIME` outcome that `sdk/wrap` makes then names that input. Without an input, the messages name only the program. A program that prints its progress on stdout, such as HandBrakeCLI, can pass the size limit in a long run. The program can succeed, but `sdk/wrap` still answers `RUNTIME`, and the decoder never runs. A handler whose decoder reads no stdout sets `DiscardStdout`. A handler that reads it sets an `OutputLimit` that its longest run cannot pass. diff --git a/sdk/i18n/locales/en.toml b/sdk/i18n/locales/en.toml index ad37aa55..904dc8d2 100644 --- a/sdk/i18n/locales/en.toml +++ b/sdk/i18n/locales/en.toml @@ -583,7 +583,7 @@ description = "Message of RUNTIME when a program that a command runs through sdk other = "{{.Name}} could not start." ["wrap.timeout"] -description = "Message of RUNTIME when a program that a command runs did not finish within the command's timeout, so ncly stopped it. {{.Name}} is the program, such as bash, and {{.Timeout}} the timeout, such as 10 s." +description = "Message of RUNTIME when a program that a command runs did not finish within the command's timeout, its version probe included, so ncly stopped it. {{.Name}} is the program, such as bash, and {{.Timeout}} the timeout, such as 10 s." other = "{{.Name}} did not finish within {{.Timeout}}, so it was stopped." ["wrap.output_over"] @@ -598,6 +598,34 @@ other = "{{.Name}} failed with exit {{.Exit}}." description = "Message of RUNTIME when a program that a command runs gave output that the command cannot read, such as invalid JSON. {{.Name}} is the program, such as bash." other = "{{.Name}} gave output that this command cannot read." +["wrap.start_failed_input"] +description = "Message of RUNTIME when a program that a command runs through sdk/wrap could not start. {{.Name}} is the program, such as bash, and {{.Input}} the item or path of the call, such as notes.md." +other = "{{.Name}} could not start for {{.Input}}." + +["wrap.timeout_input"] +description = "Message of RUNTIME when a program that a command runs did not finish within the command's timeout, its version probe included, so ncly stopped it. {{.Name}} is the program, such as bash, {{.Input}} the item or path of the call, such as notes.md, and {{.Timeout}} the timeout, such as 10 s." +other = "{{.Name}} did not finish its work on {{.Input}} within {{.Timeout}}, so it was stopped." + +["wrap.output_over_input"] +description = "Message of RUNTIME when a program that a command runs wrote more on stdout than the command reads. {{.Name}} is the program, such as bash, {{.Input}} the item or path of the call, such as notes.md, and {{.Limit}} the limit, such as 1 MB." +other = "{{.Name}} wrote more than {{.Limit}} of output for {{.Input}}." + +["wrap.failed_input"] +description = "Message of RUNTIME when a program that a command runs exited with a code that the command does not expect. {{.Name}} is the program, such as bash, {{.Input}} the item or path of the call, such as notes.md, and {{.Exit}} its exit code." +other = "{{.Name}} failed on {{.Input}} with exit {{.Exit}}." + +["wrap.malformed_input"] +description = "Message of RUNTIME when a program that a command runs gave output that the command cannot read, such as invalid JSON. {{.Name}} is the program, such as bash, and {{.Input}} the item or path of the call, such as notes.md." +other = "{{.Name}} gave output for {{.Input}} that this command cannot read." + +["wrap.interrupted"] +description = "Error when Ctrl-C stopped a command while it ran a program through sdk/wrap. {{.Input}} is the item or path of the call, such as notes.md. Finished files stay on disk." +other = "Stopped by Ctrl-C during the work on {{.Input}}. Finished work is kept." + +["wrap.terminated"] +description = "Error when the SIGTERM signal stopped a command while it ran a program through sdk/wrap. {{.Input}} is the item or path of the call, such as notes.md. Finished files stay on disk." +other = "Stopped by SIGTERM during the work on {{.Input}}. Finished work is kept." + ["worker.missing"] description = "Message of RUNTIME when an extension's code asks for a worker, an embedded script, that the extension does not embed. Only its developer can fix it. {{.Name}} is the worker, such as count.sh." other = "This extension embeds no worker named {{.Name}}." diff --git a/sdk/wrap/check.go b/sdk/wrap/check.go index bcf6a01a..a6a5fb0b 100644 --- a/sdk/wrap/check.go +++ b/sdk/wrap/check.go @@ -12,10 +12,11 @@ import ( ) // check checks the program with the rule of Required programs in -// core-spec.md. It returns where PATH found it, or the outcome that stops -// the call. That outcome is the one of a signal, or PREREQ_MISSING, whose -// hint and next hold the install command of this machine. -func check(ctx context.Context, printer *i18n.Printer, name string, r contract.Requirement, env []string, limit time.Duration) (string, *contract.Outcome) { +// core-spec.md. It returns what the check found, and the PREREQ_MISSING +// problem that stops the call, whose hint and next hold the install command +// of this machine. A signal and a found program give no problem. The +// caller decides if a probe that timed out used up the deadline of Timeout. +func check(ctx context.Context, printer *i18n.Printer, name string, r contract.Requirement, env []string, limit time.Duration) (runner.ProgramCheck, *contract.Problem) { found, err := runner.CheckProgram(ctx, name, r, env, limit) data := map[string]any{"Name": name, "Found": found.Version, "Version": r.Version, "Exit": found.Exit, "Probe": strings.Join(append([]string{name}, r.ProbeArgs()...), " ")} var message, hint string @@ -23,10 +24,8 @@ func check(ctx context.Context, printer *i18n.Printer, name string, r contract.R case err != nil: data["Exit"] = -1 message = printer.T("wrap.probe_failed", data) - case found.Signal != nil: - return "", &contract.Outcome{Signal: contract.StopSignal(found.Signal)} - case found.Verdict == runner.ProgramFound: - return found.Path, nil + case found.Signal != nil, found.Verdict == runner.ProgramFound: + return found, nil case found.Verdict == runner.ProgramMissing: message = printer.T("wrap.missing", data) case found.Verdict == runner.ProgramTooOld: @@ -43,5 +42,5 @@ func check(ctx context.Context, printer *i18n.Printer, name string, r contract.R if found.Install != nil { problem.Hint, problem.Next = strings.Join(found.Install, " "), found.Install } - return "", &contract.Outcome{Errors: []contract.Problem{problem}} + return found, &problem } diff --git a/sdk/wrap/wrap.go b/sdk/wrap/wrap.go index e36a2b97..cf013658 100644 --- a/sdk/wrap/wrap.go +++ b/sdk/wrap/wrap.go @@ -9,7 +9,12 @@ // // Pass Run the ctx of the handler, which only a signal ends, so that a // deadline of the caller never reads as a signal. Program.Timeout bounds -// the program instead. +// the program instead. A probe that uses up that deadline gives the +// timeout, not PREREQ_MISSING, because the program is there. +// +// Set Program.Input to the item or path that the call works on. Each +// RUNTIME message that Run makes then names it. The answer to a signal +// names it too. The handler cannot add the input to these outcomes. package wrap import ( @@ -57,6 +62,11 @@ type Program struct { // decoder reads no stdout, such as for an encoder that prints its // progress there. The program still writes to a pipe, as without it. DiscardStdout bool + // Input is the item or path that the call works on, such as the file + // that the handler checked. When set, the messages of the timeout, of a + // signal, and of each other RUNTIME outcome of Run and Ended name it. + // Empty keeps the messages without an input. + Input string } // Ended is how a program ended on its own, which the decoder of Run maps. @@ -69,6 +79,7 @@ type Ended struct { // Stderr holds what the program wrote on stderr, up to the limit. Stderr []byte name string + input string printer *i18n.Printer } @@ -76,26 +87,29 @@ type Ended struct { // map, such as an exit other than 0. A decoder returns it for every exit // that it does not know, so no exit code of the program passes through. func (e Ended) Failed() contract.Outcome { - data := map[string]any{"Name": e.name, "Exit": e.Exit} - return runtime(e.printer.T("wrap.failed", data), e.printer) + data := map[string]any{"Name": e.name, "Exit": e.Exit, "Input": e.input} + return runtime(e.printer.T(named("wrap.failed", e.input), data), e.printer) } // Malformed returns the RUNTIME outcome of output that the decoder cannot // read, such as missing or invalid JSON. A decoder returns it, so it never // turns a failure into a success. func (e Ended) Malformed() contract.Outcome { - return runtime(e.printer.T("wrap.malformed", map[string]any{"Name": e.name}), e.printer) + data := map[string]any{"Name": e.name, "Input": e.input} + return runtime(e.printer.T(named("wrap.malformed", e.input), data), e.printer) } // Run checks the program, then runs it from the caller's folder, and // returns what decode makes of its end. A failure that the contract defines // never reaches decode. Run answers PREREQ_MISSING for a program that is -// missing, too old, or whose probe fails or times out. It answers RUNTIME -// for a timeout or a start that fails. Without DiscardStdout, stdout over -// the limit gives RUNTIME too. A signal gives the outcome of that signal. +// missing, too old, or whose probe fails or reaches ProbeLimit. It answers +// RUNTIME for a timeout, also one during the probe, or a start that fails. +// Without DiscardStdout, stdout over the limit gives RUNTIME too. A signal +// gives the outcome of that signal. func Run(ctx context.Context, call extension.Call, p Program, decode func(Ended) contract.Outcome) contract.Outcome { printer := call.Printer - data := map[string]any{"Name": p.Name, "Path": call.Path} + data := map[string]any{"Name": p.Name, "Path": call.Path, "Input": p.Input} + fail := func(id string) contract.Outcome { return runtime(printer.T(named(id, p.Input), data), printer) } r, declared := call.Requirement(p.Name) if !declared { return runtime(printer.T("wrap.undeclared", data), printer) @@ -121,9 +135,16 @@ func Run(ctx context.Context, call extension.Call, p Program, decode func(Ended) defer cancel() probe = min(probe, p.Timeout) } - path, failed := check(ctx, printer, p.Name, r, env, probe) - if failed != nil { - return *failed + found, problem := check(ctx, printer, p.Name, r, env, probe) + switch { + case found.Signal != nil: + return stopped(found.Signal, p.Input, printer) + case found.Verdict == runner.ProgramTimedOut && p.Timeout > 0 && expired(work): + // The probe used up the deadline of Timeout, so the program is there. + data["Timeout"] = printer.Duration(p.Timeout) + return fail("wrap.timeout") + case problem != nil: + return contract.Outcome{Errors: []contract.Problem{*problem}} } limit := cmp.Or(p.OutputLimit, OutputLimit) stdout, stderr := runner.Capture{Limit: limit}, runner.Capture{Limit: limit} @@ -131,22 +152,22 @@ func Run(ctx context.Context, call extension.Call, p Program, decode func(Ended) if p.DiscardStdout { out = io.Discard } - result, err := runner.Run(work, runner.Program{Path: path, Args: p.Args, Env: env, Stdout: out, Stderr: &stderr}) + result, err := runner.Run(work, runner.Program{Path: found.Path, Args: p.Args, Env: env, Stdout: out, Stderr: &stderr}) switch { case ctx.Err() != nil: - return contract.Outcome{Signal: contract.StopSignal(runner.SignalOf(ctx))} + return stopped(runner.SignalOf(ctx), p.Input, printer) case err != nil: - return runtime(printer.T("wrap.start_failed", data), printer) + return fail("wrap.start_failed") case result.Signal != nil: // Only the deadline of Timeout stops the program when no signal // ended ctx, and the runner then stopped its whole process tree. data["Timeout"] = printer.Duration(p.Timeout) - return runtime(printer.T("wrap.timeout", data), printer) + return fail("wrap.timeout") case stdout.Over(): data["Limit"] = printer.Size(int64(limit)) - return runtime(printer.T("wrap.output_over", data), printer) + return fail("wrap.output_over") } - return decode(Ended{Exit: result.Exit, Stdout: stdout.Bytes(), Stderr: stderr.Bytes(), name: p.Name, printer: printer}) + return decode(Ended{Exit: result.Exit, Stdout: stdout.Bytes(), Stderr: stderr.Bytes(), name: p.Name, input: p.Input, printer: printer}) } // runtime returns the RUNTIME outcome of a message, whose hint warns that @@ -154,3 +175,35 @@ func Run(ctx context.Context, call extension.Call, p Program, decode func(Ended) func runtime(message string, printer *i18n.Printer) contract.Outcome { return contract.Outcome{Errors: []contract.Problem{{Code: contract.Runtime, Message: message, Hint: printer.T("outcome.check_hint")}}} } + +// named returns the catalog ID of a message that names the input, or id +// itself when the handler set no input. +func named(id, input string) string { + if input == "" { + return id + } + return id + "_input" +} + +// expired reports whether the deadline of ctx has passed. It reads the +// clock, because ctx may end a moment after its deadline. +func expired(ctx context.Context) bool { + deadline, ok := ctx.Deadline() + return ok && !time.Now().Before(deadline) +} + +// stopped returns the outcome of a signal. With an input, its error names +// that input, and Finish puts this error first, because its code leads. +func stopped(sig os.Signal, input string, printer *i18n.Printer) contract.Outcome { + o := contract.Outcome{Signal: contract.StopSignal(sig)} + if input == "" { + return o + } + code, id := contract.Interrupted, "wrap.interrupted" + if o.Signal == contract.Terminate { + code, id = contract.Terminated, "wrap.terminated" + } + message := printer.T(id, map[string]any{"Input": input}) + o.Errors = []contract.Problem{{Code: code, Message: message, Hint: printer.T("outcome.check_hint")}} + return o +} diff --git a/testdata/extensions/hello/count.go b/testdata/extensions/hello/count.go index 1c372899..d0c2ceac 100644 --- a/testdata/extensions/hello/count.go +++ b/testdata/extensions/hello/count.go @@ -14,7 +14,8 @@ import ( // count counts the lines of a file with the worker count.sh, which bash // runs through sdk/wrap. It answers with the number that the worker prints -// as JSON, and checks the file before the worker starts. +// as JSON, and checks the file before the worker starts. The file is the +// input of sdk/wrap, so each failure that sdk/wrap answers names it. func count(ctx context.Context, call extension.Call) contract.Outcome { p := call.Printer file := call.Args[0] @@ -35,7 +36,7 @@ func count(ctx context.Context, call extension.Call) contract.Outcome { if problem != nil { return contract.Outcome{Errors: []contract.Problem{*problem}} } - program := wrap.Program{Name: "bash", Args: []string{worker, file}, Timeout: timeout} + program := wrap.Program{Name: "bash", Args: []string{worker, file}, Timeout: timeout, Input: file} return wrap.Run(ctx, call, program, func(end wrap.Ended) contract.Outcome { var answer struct { Lines *int `json:"lines"` diff --git a/testdata/extensions/hello/testdata/script/wrap.txtar b/testdata/extensions/hello/testdata/script/wrap.txtar index 31e42b87..fc2e26ce 100644 --- a/testdata/extensions/hello/testdata/script/wrap.txtar +++ b/testdata/extensions/hello/testdata/script/wrap.txtar @@ -53,37 +53,39 @@ answer stderr USAGE_INVALID ! exists worker.path # An exit other than 0, even the program's own 75, is RUNTIME with exit 1, and so is output -# that the handler cannot read or that is over the limit of 1 MB +# that the handler cannot read or that is over the limit of 1 MB. hello count sets the input +# of sdk/wrap to its file, so each message names that file env STUB=fail exits 1 ncly hello count input.md --json ! stdout . answer stderr RUNTIME -stderr '"message":"bash failed with exit 3\.","hint":"Check the results before you run anything again\."\}' +stderr '"message":"bash failed on input\.md with exit 3\.","hint":"Check the results before you run anything again\."\}' exits 1 ncly --lang en-XA hello count input.md -pseudo stderr bash +pseudo stderr bash input.md env STUB=retry exits 1 ncly hello count input.md --json answer stderr RUNTIME -stderr '"message":"bash failed with exit 75\.' +stderr '"message":"bash failed on input\.md with exit 75\.' env STUB=garbled exits 1 ncly hello count input.md --json answer stderr RUNTIME -stderr '"message":"bash gave output that this command cannot read\.' +stderr '"message":"bash gave output for input\.md that this command cannot read\.' env STUB=noisy exits 1 ncly hello count input.md --json answer stderr RUNTIME -stderr '"message":"bash wrote more than 1 MB of output\.' +stderr '"message":"bash wrote more than 1 MB of output for input\.md\.' # A program that prints more than 1 MB of progress on stdout, as an encoder does in a long # run, then exits 0, succeeds for hello scan, which sets DiscardStdout. hello count keeps its -# stdout, so the same progress still fails it. Discarded stdout keeps each exit mapped +# stdout, so the same progress still fails it. Discarded stdout keeps each exit mapped. +# hello scan sets no input, so its message names only the program env STUB=progress exec ncly hello scan input.md --json stdout '\A\{"ok":true,"contract_version":1,"file":"input\.md"\}\n\z' ! stderr . exits 1 ncly hello count input.md --json answer stderr RUNTIME -stderr '"message":"bash wrote more than 1 MB of output\.' +stderr '"message":"bash wrote more than 1 MB of output for input\.md\.' env STUB=fail exits 1 ncly hello scan input.md --json ! stdout . diff --git a/testdata/extensions/hello/testdata/script/wrap_stop.txtar b/testdata/extensions/hello/testdata/script/wrap_stop.txtar index 78fdf3bc..6797cf51 100644 --- a/testdata/extensions/hello/testdata/script/wrap_stop.txtar +++ b/testdata/extensions/hello/testdata/script/wrap_stop.txtar @@ -1,6 +1,7 @@ # hello count runs its worker count.sh with bash through sdk/wrap, as wrap.txtar shows. This # scenario covers what stops the program: the timeout, Ctrl-C, and a version probe that does -# not finish. stub/bash stands in for bash. It answers the version probe, writes the path of +# not finish. hello count sets its file as the input of sdk/wrap, and hello scan sets none. +# stub/bash stands in for bash. It answers the version probe, writes the path of # the worker, then does what STUB says. chmod 755 stub/bash env PATH=$WORK/stub:$PATH @@ -11,12 +12,13 @@ env LANG= LC_ALL= LC_MESSAGES= NCLY_LANG= env STUB=hang exits 1 ncly hello count input.md --timeout 1s --json answer stderr RUNTIME -stderr '"message":"bash did not finish within 1 s, so it was stopped\.' +stderr '"message":"bash did not finish its work on input\.md within 1 s, so it was stopped\.' gone hang.pid gone child.pid exec /bin/sh -c 'test ! -e "$(cat worker.path)"' -# Ctrl-C stops the process tree too, with exit 130, and the folder of the worker goes +# Ctrl-C stops the process tree too, with exit 130, and the folder of the worker goes. The +# answer names the input rm hang.pid child.pid worker.path exec exitcode exit.txt ncly hello count input.md --json & waitfile hang.pid @@ -25,10 +27,34 @@ wait grep '^130$' exit.txt ! stdout . answer stderr INTERRUPTED +stderr '"errors":\[\{"code":"INTERRUPTED","message":"Stopped by Ctrl-C during the work on input\.md\. Finished work is kept\.","hint":"Check the results before you run anything again\."\}\]' gone hang.pid gone child.pid exec /bin/sh -c 'test ! -e "$(cat worker.path)"' +# SIGTERM stops it the same way, with exit 143, and the answer names the input +rm hang.pid child.pid worker.path +exec exitcode exit.txt ncly hello count input.md --json & +waitfile hang.pid +term +wait +grep '^143$' exit.txt +answer stderr TERMINATED +stderr '"errors":\[\{"code":"TERMINATED","message":"Stopped by SIGTERM during the work on input\.md\. Finished work is kept\.' +gone hang.pid +gone child.pid + +# A command that sets no input keeps the message of the signal alone +rm hang.pid child.pid +exec exitcode exit.txt ncly hello scan input.md --json & +waitfile hang.pid +kill -INT +wait +grep '^130$' exit.txt +answer stderr INTERRUPTED +stderr '"message":"Stopped by Ctrl-C\. Finished work is kept\."' +gone hang.pid + # A program that ignores Ctrl-C runs until the SIGKILL of core's cleanup, 4 seconds in the # test binary, and the folder of the worker is gone before that SIGKILL rm hang.pid child.pid worker.path @@ -42,13 +68,26 @@ answer stderr INTERRUPTED gone stubborn.pid exec /bin/sh -c 'test ! -e "$(cat worker.path)"' -# A version probe that does not finish in time is PREREQ_MISSING with exit 78, before work +# A version probe that uses up the timeout of the command gives the timeout, with exit 1, +# before work. So does a timeout too short for any probe rm worker.path env STUB=probe-hang +exits 1 ncly hello count input.md --timeout 1s --json +answer stderr RUNTIME +stderr '"message":"bash did not finish its work on input\.md within 1 s, so it was stopped\.' +gone probe.pid +! exists worker.path +env STUB= +exits 1 ncly hello count input.md --timeout 1ms --json +answer stderr RUNTIME +stderr '"message":"bash did not finish its work on input\.md within 1 ms, so it was stopped\.' +! exists worker.path + +# A version probe that fails is PREREQ_MISSING with exit 78, before work +env STUB=probe-fail exits 78 ncly hello count input.md --timeout 1s --json answer stderr PREREQ_MISSING -stderr '"message":"bash --version did not print its version in time\.","hint":"Check that bash --version prints a version and exits\."\}' -gone probe.pid +stderr '"message":"bash --version exited with 2, so its version is unknown\."' ! exists worker.path -- input.md -- @@ -60,6 +99,7 @@ if [ "$1" = --version ]; then echo $$ > "$WORK/probe.pid" exec /bin/sleep 30 fi + [ "$STUB" = probe-fail ] && exit 2 echo 'GNU bash, version 5.2.0(1)-release' exit 0 fi From 3fc33c043b5f9b2748fc888b29a1ec6f6775527d Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:43:06 -0400 Subject: [PATCH 12/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20inspect?= =?UTF-8?q?=20names=20its=20file=20in=20signal=20and=20timeout=20answers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inspect example sets its path as the input of sdk/wrap. Its scenario checks that the answers to a failed exit, the timeout, Ctrl-C, and SIGTERM name the file. A version probe that uses up the timeout now expects the timeout, RUNTIME with exit 1. A probe that fails, or a python3 that is too old, stays PREREQ_MISSING. spec.md moves the probe that uses up the timeout to worker-fails. Its no-python line drops the slow probe, which no scenario proves. Refs #407 Co-Authored-By: Claude Opus 5.5 --- template/inspect.go.jinja | 10 ++++---- template/spec.md.jinja | 4 ++-- template/testdata/script/inspect.txtar.jinja | 25 +++++++++++++------- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/template/inspect.go.jinja b/template/inspect.go.jinja index 9ec68594..4b3b1e23 100644 --- a/template/inspect.go.jinja +++ b/template/inspect.go.jinja @@ -16,9 +16,11 @@ import ( // inspect gives the size and the line count of a file. python3 runs the // worker inspect.py through sdk/wrap, which answers each failure that Wrapped // programs in the contract lists. The handler checks its input before the -// worker starts. It never maps an exit other than 0, or output that it cannot -// read, to a success. It also refuses a file that the worker accepts but -// whose lines mean nothing, as testdata/script/refusal.txtar shows. +// worker starts. The file is the input of sdk/wrap, so each failure that +// sdk/wrap answers, a timeout or a signal included, names it. The handler +// never maps an exit other than 0, or output that it cannot read, to a +// success. It also refuses a file that the worker accepts but whose lines +// mean nothing, as testdata/script/refusal.txtar shows. func inspect(ctx context.Context, call extension.Call) contract.Outcome { p := call.Printer file := call.Args[0] @@ -39,7 +41,7 @@ func inspect(ctx context.Context, call extension.Call) contract.Outcome { if problem != nil { return contract.Outcome{Errors: []contract.Problem{*problem}} } - program := wrap.Program{Name: "python3", Args: []string{worker, file}, Timeout: timeout} + program := wrap.Program{Name: "python3", Args: []string{worker, file}, Timeout: timeout, Input: file} return wrap.Run(ctx, call, program, func(end wrap.Ended) contract.Outcome { var answer struct { Size *int64 `json:"size_bytes"` diff --git a/template/spec.md.jinja b/template/spec.md.jinja index dc8fb23b..fc6a0400 100644 --- a/template/spec.md.jinja +++ b/template/spec.md.jinja @@ -33,8 +33,8 @@ The dry run makes the same free inspection and gives the same result. | ID | Condition | Code | |---|---|---| | `bad-input` | The file is missing or is no regular file, or `--timeout` is no duration | `USAGE_INVALID` | -| `no-python` | `python3` is missing or older than 3.9, or its version probe fails or does not finish in time | `PREREQ_MISSING` | -| `worker-fails` | `python3` exits with a code other than 0, its output is missing, malformed, or over 1 MB, or the timeout expires | `RUNTIME` | +| `no-python` | `python3` is missing or older than 3.9, or its version probe fails | `PREREQ_MISSING` | +| `worker-fails` | `python3` exits with a code other than 0, its output is missing, malformed, or over 1 MB, or the timeout expires, even during the probe | `RUNTIME` | | `interrupted` | Ctrl-C arrives | `INTERRUPTED` | | `terminated` | SIGTERM arrives | `TERMINATED` | diff --git a/template/testdata/script/inspect.txtar.jinja b/template/testdata/script/inspect.txtar.jinja index 78908f09..802913b9 100644 --- a/template/testdata/script/inspect.txtar.jinja +++ b/template/testdata/script/inspect.txtar.jinja @@ -35,14 +35,14 @@ stderr 'missing.md is no file that {{ root }} can inspect[.]' exits 2 ncly {{ example_command }} notes/input.md --timeout soon --json stderr '"code":"USAGE_INVALID"' -# python3 too old, or a version probe that does not finish in time, is -# PREREQ_MISSING with exit 78, before the worker starts. +# python3 too old, or a version probe that fails, is PREREQ_MISSING with +# exit 78, before the worker starts. # spec: no-python env STUB=old exits 78 ncly {{ example_command }} notes/input.md --json stderr '"code":"PREREQ_MISSING"' -env STUB=probe-hang -exits 78 ncly {{ example_command }} notes/input.md --timeout 1s --json +env STUB=probe-fail +exits 78 ncly {{ example_command }} notes/input.md --json stderr '"code":"PREREQ_MISSING"' # An exit other than 0, even the program's own 75, is RUNTIME with exit 1, and @@ -50,7 +50,7 @@ stderr '"code":"PREREQ_MISSING"' # spec: worker-fails env STUB=fail exits 1 ncly {{ example_command }} notes/input.md --json -stderr '"code":"RUNTIME"' +stderr '"code":"RUNTIME","message":"python3 failed on notes/input[.]md with exit 3[.]"' env STUB=retry exits 1 ncly {{ example_command }} notes/input.md --json stderr '"code":"RUNTIME"' @@ -61,17 +61,24 @@ env STUB=noisy exits 1 ncly {{ example_command }} notes/input.md --json stderr '"code":"RUNTIME"' -# At the timeout, the program stops, and the answer is RUNTIME with exit 1. +# At the timeout, the program stops, and the answer is RUNTIME with exit 1. A +# version probe that uses up the timeout gives the same answer. Each names +# the file. # spec: worker-fails +env STUB=probe-hang +exits 1 ncly {{ example_command }} notes/input.md --timeout 1s --json +stderr '"code":"RUNTIME","message":"python3 did not finish its work on notes/input[.]md within 1 s, so it was stopped[.]"' env STUB=hang exits 1 ncly {{ example_command }} notes/input.md --timeout 1s --json -stderr '"code":"RUNTIME"' +stderr '"code":"RUNTIME","message":"python3 did not finish its work on notes/input[.]md within 1 s, so it was stopped[.]"' -# Ctrl-C stops the program too, and the answer is INTERRUPTED, whose exit is 130. +# Ctrl-C stops the program too, and the answer is INTERRUPTED, whose exit is +# 130. Its message names the file. # spec: interrupted rm hang.pid survives finished.md signal INT hang.pid ncly {{ example_command }} notes/input.md --json +stderr '"code":"INTERRUPTED","message":"Stopped by Ctrl-C during the work on notes/input[.]md[.]' # SIGTERM stops it the same way, and the answer is TERMINATED, whose exit is # 143. The kit reads the exit of the host, after its cleanup finishes. @@ -80,6 +87,7 @@ rm hang.pid rm finished.md survives finished.md signal TERM hang.pid ncly {{ example_command }} notes/input.md --json +stderr '"code":"TERMINATED","message":"Stopped by SIGTERM during the work on notes/input[.]md[.]' # python3 missing from PATH is PREREQ_MISSING with exit 78, before any work. # spec: no-python @@ -100,6 +108,7 @@ case "$1:$STUB" in --version:) exec /usr/bin/python3 --version ;; --version:old) echo 'Python 2.7.18'; exit 0 ;; --version:probe-hang) exec sleep 30 ;; +--version:probe-fail) exit 2 ;; --version:*) echo 'Python 3.12.0'; exit 0 ;; esac case "$STUB" in From 95e94df99456efbd9a0863fbdcb3f91892084bbb Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:41:37 -0400 Subject: [PATCH 13/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20contract:=20move?= =?UTF-8?q?=20the=20scenario=20commands=20of=20the=20test=20kit=20to=20the?= =?UTF-8?q?ir=20own=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test-kit.md is near the limit of 2,500 tokens. The section moves without a change of text, so later rules about the kit commands fit. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract.md | 1 + docs/north-star/contract/scenario-commands.md | 45 +++++++++++++++++++ docs/north-star/contract/test-kit.md | 41 +---------------- 3 files changed, 47 insertions(+), 40 deletions(-) create mode 100644 docs/north-star/contract/scenario-commands.md diff --git a/docs/north-star/contract.md b/docs/north-star/contract.md index af7d461d..f6231a05 100644 --- a/docs/north-star/contract.md +++ b/docs/north-star/contract.md @@ -45,6 +45,7 @@ Each part is one file, in reading order. - [Wrapped programs](contract/wrap.md) - [Test kit](contract/test-kit.md) - [Manifest examples](contract/manifest-examples.md) + - [Scenario commands](contract/scenario-commands.md) - [Spec IDs](contract/spec-ids.md) - [Generic cases](contract/generic-cases.md) - [Extension repository](contract/extension-repository.md) diff --git a/docs/north-star/contract/scenario-commands.md b/docs/north-star/contract/scenario-commands.md new file mode 100644 index 00000000..088cbd77 --- /dev/null +++ b/docs/north-star/contract/scenario-commands.md @@ -0,0 +1,45 @@ +# Scenario commands + +This part says which commands a scenario of an extension repository uses, and what each command of the kit checks. +[Test kit](test-kit.md) says how the kit runs the scenarios. + +A scenario uses the commands of [testscript](https://pkg.go.dev/github.com/rogpeppe/go-internal/testscript), and these ones of the kit: + +| Command | What it does | +|---|---| +| `ncly ...` | Runs `ncly`, which exits 0, or another code after `!`, and checks the answer | +| `ncly- ...` | Runs the extension directly, the same way | +| `exits ...` | Runs a command, these two included, and checks its exact exit code | +| `signal INT\|TERM ...` | Waits for a new marker, signals the host, and checks its exit and answer | +| `survives ...` | Requires these finished files at the next signal's marker and unchanged after its exit | +| `protect ...` | Selects files, folders, or absent paths that subsequent `refuses` calls must keep unchanged | +| `refuses ...` | Requires the same refusal code and exit in a dry run and a run | +| `grant ` | Grants the key of a declared service to the extension at its scratch path | +| `fake ` | Puts the fake child in `$WORK/bin`, first in `PATH`, under the name `name` | + +The fake child runs the actions of `$WORK/bin/.fake`, one per line: `stdout `, `stderr `, `stdin`, `env`, `sleep `, and `exit `. `env` writes the names of its variables, never a value. The package comment of `sdk/extensiontest/fake` defines each action. + +`signal` and `refuses` accept only registered `ncly` and `ncly-` calls of runnable commands. +They enable JSON and non-interactive mode, and validate each answer through `Coverage.Observe`. +Explicit flags that disable JSON or non-interactive mode are refused. +Both commands leave the last call's streams available to `stdout` and `stderr` assertions. +Their `ncly` calls use the registered host's absolute path, even when a scenario replaces `PATH`. + +`signal` runs the host in the foreground, without a shell cleanup command supplying its result. +Its marker must not exist before the call. +It waits up to 10 seconds for a regular marker file, then sends the signal to the host process. +The exit deadline allows `runner.Cleanup`, `runner.DrainLimit`, and a three-second scheduling margin after the signal. +The host starts in its own process group, which the kit kills on timeout or failure. +Stream draining stops within `runner.DrainLimit` after the host exits, even when a child keeps a stream open. +INT requires exit 130 and `INTERRUPTED`; TERM requires exit 143 and `TERMINATED`. +`survives` selects files for one signal call, including files that the call finishes before its marker. +Their bytes and permissions must remain unchanged after the host exits. + +`refuses` requires a command with a declared dry run and arguments without dry-run flags. +It adds `--dry-run` for the first call and uses the original arguments for the second call. +Both calls receive the same stdin bytes. +Both calls must fail with the requested leading error code and the same nonzero exit code. +Each call must preserve the protected paths, including absent paths and each folder's contents. +The comparison records symbolic links without following them. +Nicely's cache stays unprotected unless a scenario selects its path. +Select inputs and possible outputs with `protect` before each refusal group. diff --git a/docs/north-star/contract/test-kit.md b/docs/north-star/contract/test-kit.md index 3595db3b..288a7c07 100644 --- a/docs/north-star/contract/test-kit.md +++ b/docs/north-star/contract/test-kit.md @@ -22,46 +22,7 @@ A scratch `HOME` does not hide the OS keychain. `ncly` reads it only when the va ## Scenario commands -A scenario uses the commands of [testscript](https://pkg.go.dev/github.com/rogpeppe/go-internal/testscript), and these ones of the kit: - -| Command | What it does | -|---|---| -| `ncly ...` | Runs `ncly`, which exits 0, or another code after `!`, and checks the answer | -| `ncly- ...` | Runs the extension directly, the same way | -| `exits ...` | Runs a command, these two included, and checks its exact exit code | -| `signal INT\|TERM ...` | Waits for a new marker, signals the host, and checks its exit and answer | -| `survives ...` | Requires these finished files at the next signal's marker and unchanged after its exit | -| `protect ...` | Selects files, folders, or absent paths that subsequent `refuses` calls must keep unchanged | -| `refuses ...` | Requires the same refusal code and exit in a dry run and a run | -| `grant ` | Grants the key of a declared service to the extension at its scratch path | -| `fake ` | Puts the fake child in `$WORK/bin`, first in `PATH`, under the name `name` | - -The fake child runs the actions of `$WORK/bin/.fake`, one per line: `stdout `, `stderr `, `stdin`, `env`, `sleep `, and `exit `. `env` writes the names of its variables, never a value. The package comment of `sdk/extensiontest/fake` defines each action. - -`signal` and `refuses` accept only registered `ncly` and `ncly-` calls of runnable commands. -They enable JSON and non-interactive mode, and validate each answer through `Coverage.Observe`. -Explicit flags that disable JSON or non-interactive mode are refused. -Both commands leave the last call's streams available to `stdout` and `stderr` assertions. -Their `ncly` calls use the registered host's absolute path, even when a scenario replaces `PATH`. - -`signal` runs the host in the foreground, without a shell cleanup command supplying its result. -Its marker must not exist before the call. -It waits up to 10 seconds for a regular marker file, then sends the signal to the host process. -The exit deadline allows `runner.Cleanup`, `runner.DrainLimit`, and a three-second scheduling margin after the signal. -The host starts in its own process group, which the kit kills on timeout or failure. -Stream draining stops within `runner.DrainLimit` after the host exits, even when a child keeps a stream open. -INT requires exit 130 and `INTERRUPTED`; TERM requires exit 143 and `TERMINATED`. -`survives` selects files for one signal call, including files that the call finishes before its marker. -Their bytes and permissions must remain unchanged after the host exits. - -`refuses` requires a command with a declared dry run and arguments without dry-run flags. -It adds `--dry-run` for the first call and uses the original arguments for the second call. -Both calls receive the same stdin bytes. -Both calls must fail with the requested leading error code and the same nonzero exit code. -Each call must preserve the protected paths, including absent paths and each folder's contents. -The comparison records symbolic links without following them. -Nicely's cache stays unprotected unless a scenario selects its path. -Select inputs and possible outputs with `protect` before each refusal group. +[Scenario commands](scenario-commands.md) lists the commands of a scenario, and says what each command of the kit checks. ## What it checks From b82bcfaf6e1332c77cd221a1b37200adb99f1107 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:44:43 -0400 Subject: [PATCH 14/61] =?UTF-8?q?=E2=9C=A8=20feat:=20sdk/extensiontest:=20?= =?UTF-8?q?protect=20also=20guards=20the=20next=20signal=20call?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An interrupted run must keep its input, leave an absent output absent, and keep an old output. A protect selection now applies to the next signal call, which ends it for signal. refuses keeps the selection until the next protect call, as before. The comparison skips the marker. signal and the coming appears command share the foreground run, the wait for the marker, and the bounded wait for the exit. A kit test also proves that protect and refuses catch a run that writes through a link. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/scenario-commands.md | 12 +- sdk/extensiontest/extensiontest.go | 2 +- sdk/extensiontest/paired.go | 10 +- sdk/extensiontest/protect_test.go | 134 ++++++++++++++++++ sdk/extensiontest/signal.go | 112 ++++++++++----- 5 files changed, 232 insertions(+), 38 deletions(-) create mode 100644 sdk/extensiontest/protect_test.go diff --git a/docs/north-star/contract/scenario-commands.md b/docs/north-star/contract/scenario-commands.md index 088cbd77..4338dc1a 100644 --- a/docs/north-star/contract/scenario-commands.md +++ b/docs/north-star/contract/scenario-commands.md @@ -12,7 +12,7 @@ A scenario uses the commands of [testscript](https://pkg.go.dev/github.com/rogpe | `exits ...` | Runs a command, these two included, and checks its exact exit code | | `signal INT\|TERM ...` | Waits for a new marker, signals the host, and checks its exit and answer | | `survives ...` | Requires these finished files at the next signal's marker and unchanged after its exit | -| `protect ...` | Selects files, folders, or absent paths that subsequent `refuses` calls must keep unchanged | +| `protect ...` | Selects files, folders, or absent paths that the next `signal` call and subsequent `refuses` calls must keep unchanged | | `refuses ...` | Requires the same refusal code and exit in a dry run and a run | | `grant ` | Grants the key of a declared service to the extension at its scratch path | | `fake ` | Puts the fake child in `$WORK/bin`, first in `PATH`, under the name `name` | @@ -43,3 +43,13 @@ Each call must preserve the protected paths, including absent paths and each fol The comparison records symbolic links without following them. Nicely's cache stays unprotected unless a scenario selects its path. Select inputs and possible outputs with `protect` before each refusal group. + +A `protect` selection applies to each later `refuses` call, until the next `protect` call replaces it. +It also applies to the next `signal` call, and that call ends it for `signal`. +Give a new `protect` call before each `signal` call that must keep paths. +`signal` compares the protected paths before the host starts and after it exits, except its marker. +So an interrupted run must keep its input, leave an absent output absent, and keep an old output. + +To prove that a command never writes through a link, make its output a symbolic link to its input. +Protect both paths, then call `refuses`. +A write through the link changes the input, and a replacement of the link changes the link. diff --git a/sdk/extensiontest/extensiontest.go b/sdk/extensiontest/extensiontest.go index 95cb4254..4e0d644e 100644 --- a/sdk/extensiontest/extensiontest.go +++ b/sdk/extensiontest/extensiontest.go @@ -14,7 +14,7 @@ // exits runs a command, these two included, and checks its exact exit code // signal INT|TERM checks the host's signal exit and answer // survives ... requires finished files before and after the next signal -// protect ... selects paths that subsequent refusal pairs must preserve +// protect ... selects paths that the next signal and later refusal pairs must preserve // refuses checks the same refusal in a dry run and a run // grant grants the key of a declared service to the extension // fake puts the fake child in $WORK/bin as name diff --git a/sdk/extensiontest/paired.go b/sdk/extensiontest/paired.go index 0cf27472..85042510 100644 --- a/sdk/extensiontest/paired.go +++ b/sdk/extensiontest/paired.go @@ -14,8 +14,10 @@ import ( "github.com/rogpeppe/go-internal/testscript" ) +// scriptFiles holds the selections of one scenario. +// protect lasts until the next protect call, and next until the next signal call. type scriptFiles struct { - protect, survives []string + protect, next, survives []string } // ScenarioCommands supplies signal, refuses, protect, and survives to a testscript runner. @@ -48,7 +50,7 @@ func (c *Coverage) ScenarioCommands(host string) map[string]func(*testscript.Tes if survives { state(ts).survives = paths } else { - state(ts).protect = paths + state(ts).protect, state(ts).next = paths, paths } } } @@ -56,8 +58,10 @@ func (c *Coverage) ScenarioCommands(host string) map[string]func(*testscript.Tes "protect": selectFiles(false), "survives": selectFiles(true), "signal": func(ts *testscript.TestScript, neg bool, args []string) { files := state(ts) + next := files.next + files.next = nil defer func() { files.survives = nil }() - c.signal(ts, neg, args, files.survives, host) + c.signal(ts, neg, args, files.survives, next, host) }, "refuses": func(ts *testscript.TestScript, neg bool, args []string) { c.refuses(ts, neg, args, state(ts).protect, host) diff --git a/sdk/extensiontest/protect_test.go b/sdk/extensiontest/protect_test.go new file mode 100644 index 00000000..ff1e007d --- /dev/null +++ b/sdk/extensiontest/protect_test.go @@ -0,0 +1,134 @@ +package extensiontest_test + +import ( + "os" + "strings" + "testing" +) + +// copyCommand adds standalone copy, a command that writes a text output, to the standalone fixture. +// Each scenario replaces the installed extension with a shell host, as pairedFixture does. +const copyCommand = `[[commands]] +path = "standalone copy" +summary = "standalone.tally.summary" +effects = { user_writes = true } +modes = { dry_run = true } + +[[commands.args]] +name = "input" +summary = "standalone.tally.summary" +required = true + +[[commands.args]] +name = "output" +summary = "standalone.tally.summary" +required = true + +[commands.result.schema] +type = "object" +required = ["output"] +properties = { output = { type = "string" } } + +[commands.result.dry_run_schema] +type = "object" +required = ["output"] +properties = { output = { type = "string" } } + +[programs.tally-tool]` + +const copyRefusal = `echo '{"ok":false,"contract_version":1,"retry_safe":true,"errors":[{"code":"USAGE_INVALID","message":"notes.txt exists.","hint":"Add --force."}]}' >&2 +exit 2 +` + +func copyFixture(t *testing.T, host, script string) (string, error) { + t.Helper() + ext := "$XDG_DATA_HOME/nicely/extensions/ncly-standalone" + setup := "mkdir out\nrm " + ext + "\ncp host.sh " + ext + "\nchmod 755 " + ext + "\n" + return goTest(t, "standalone", os.Getenv("PATH"), + edit{"manifest.toml", "[programs.tally-tool]", copyCommand}, + edit{"extension.go", `"standalone fetch": fetch,`, "\"standalone fetch\": fetch,\n\t\t\"standalone copy\": tally,"}, + edit{file: "testdata/script/copy.txtar", new: setup + script + "\n-- notes.md --\nFirst line\nLast line\n-- host.sh --\n" + host}) +} + +// signalCopyHost copies its input to a temporary file, writes the marker, and waits. +// On SIGTERM, it runs cleanup and answers TERMINATED. +func signalCopyHost(cleanup string) string { + return `#!/bin/sh +[ "$1" = copy ] && shift +input=$1 output=$2 tmp=$(dirname "$2")/.copy.$$ +cat "$input" > "$tmp" +trap '` + cleanup + `; echo '\''{"ok":false,"contract_version":1,"retry_safe":true,"errors":[{"code":"TERMINATED","message":"Stopped.","hint":"Run the command again."}]}'\'' >&2; exit 143' TERM +echo ready > started +while :; do sleep 0.01; done +` +} + +// TestProtectSignal proves that a protect selection applies to the next signal call. +// The interrupted run keeps the input, leaves an absent output absent, and keeps an old output. +func TestProtectSignal(t *testing.T) { + clean := signalCopyHost(`rm -f "$tmp"`) + call := "signal TERM started ncly-standalone copy notes.md out/notes.txt\n" + again := "rm started\n" + call + old := "cp notes.md out/notes.txt\n" + cases := []struct{ name, host, script, problem string }{ + {"clean interruption", clean, "protect notes.md out\n" + call, ""}, + {"marker in a protected folder", clean, "protect notes.md .\n" + call, ""}, + {"temporary file left", signalCopyHost(":"), "protect notes.md out\n" + call, "signal: TERM changed protected paths"}, + {"partial output", signalCopyHost(`mv "$tmp" "$output"`), "protect notes.md out/notes.txt\n" + call, "signal: TERM changed protected paths"}, + {"old output changed", signalCopyHost(`rm -f "$tmp"; echo partial > "$output"`), old + "protect notes.md out/notes.txt\n" + call, "signal: TERM changed protected paths"}, + {"input changed", signalCopyHost(`rm -f "$tmp"; echo more >> "$input"`), "protect notes.md\n" + call, "signal: TERM changed protected paths"}, + {"selection ends after one signal", signalCopyHost(`rm -f "$tmp"; [ -z "$DIRTY" ] || echo partial > "$output"`), "protect notes.md out\n" + call + "env DIRTY=1\n" + again, ""}, + {"new selection for each signal", signalCopyHost(`rm -f "$tmp"; [ -z "$DIRTY" ] || echo partial > "$output"`), "protect notes.md out\n" + call + "env DIRTY=1\nprotect notes.md out\n" + again, "signal: TERM changed protected paths"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + out, err := copyFixture(t, c.host, c.script) + if c.problem == "" { + if err != nil { + t.Fatalf("valid signal scenario failed: %v\n%s", err, out) + } + } else if err == nil || !strings.Contains(out, c.problem) { + t.Fatalf("want %q, got %v\n%s", c.problem, err, out) + } + }) + } +} + +// linkHost refuses an output that is a link. Before the refusal, the run part runs write. +func linkHost(write string) string { + return `#!/bin/sh +dry_run=false +if [ "$1" = --dry-run=true ]; then dry_run=true; shift; fi +[ "$1" = copy ] && shift +input=$1 output=$2 +[ "$dry_run" = true ] || { ` + write + `; } +` + copyRefusal +} + +// TestProtectLink proves that protect and refuses catch a run that writes through a link. +// The output is a symbolic link to the input, and the protection records the link without following it. +func TestProtectLink(t *testing.T) { + ext := "$XDG_DATA_HOME/nicely/extensions/ncly-standalone" + refusal := "symlink out/notes.txt -> ../notes.md\nprotect notes.md out/notes.txt\nrefuses USAGE_INVALID ncly-standalone copy notes.md out/notes.txt\n" + cases := []struct{ name, host, script, problem string }{ + {"refusal keeps the link", linkHost(":"), refusal, ""}, + {"write through the link", linkHost(`echo copy >> "$output"`), refusal, "run changed protected paths"}, + {"truncate through the link", linkHost(`: > "$output"`), refusal, "run changed protected paths"}, + {"replace the link", linkHost(`rm "$output"; cp "$input" "$output"`), refusal, "run changed protected paths"}, + {"selection stays after a signal", linkHost(`echo copy >> "$output"`), "symlink out/notes.txt -> ../notes.md\nprotect notes.md out/notes.txt\ncp signal.sh " + ext + "\n" + + "signal TERM started ncly-standalone copy notes.md out/other.txt\ncp host.sh " + ext + "\nrefuses USAGE_INVALID ncly-standalone copy notes.md out/notes.txt\n" + + "-- signal.sh --\n" + signalCopyHost(`rm -f "$tmp"`), "run changed protected paths"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + out, err := copyFixture(t, c.host, c.script) + if c.problem == "" { + if err != nil { + t.Fatalf("valid link scenario failed: %v\n%s", err, out) + } + } else if err == nil || !strings.Contains(out, c.problem) { + t.Fatalf("want %q, got %v\n%s", c.problem, err, out) + } + }) + } +} diff --git a/sdk/extensiontest/signal.go b/sdk/extensiontest/signal.go index 40213e56..b4f6472d 100644 --- a/sdk/extensiontest/signal.go +++ b/sdk/extensiontest/signal.go @@ -16,7 +16,7 @@ import ( "github.com/rogpeppe/go-internal/testscript" ) -func (c *Coverage) signal(ts *testscript.TestScript, neg bool, args, survives []string, registeredHost string) { +func (c *Coverage) signal(ts *testscript.TestScript, neg bool, args, survives, protected []string, registeredHost string) { if neg || len(args) < 3 || (args[0] != "INT" && args[0] != "TERM") { ts.Fatalf("usage: signal INT|TERM ...") } @@ -26,19 +26,48 @@ func (c *Coverage) signal(ts *testscript.TestScript, neg bool, args, survives [] if _, err := os.Lstat(marker); !os.IsNotExist(err) { ts.Fatalf("signal: marker %s must not exist before the call", args[1]) } - dir, err := os.MkdirTemp("", "ncly-signal-") + before, err := snapshotFiles(protected, false) + ts.Check(err) + sig, wantExit, wantCode := syscall.SIGINT, 130, contract.Interrupted + if args[0] == "TERM" { + sig, wantExit, wantCode = syscall.SIGTERM, 143, contract.Terminated + } + exit, stdout, stderr := foreground(ts, "signal", program, args[3:], func(done <-chan struct{}, host *os.Process) error { + return sendSignal(done, host, marker, survives, sig) + }) + code, problem := c.checkedCode(words, exit, stdout, stderr) + if exit != wantExit { + ts.Fatalf("signal: %s exited with %d, want %d", args[2], exit, wantExit) + } + if problem != "" { + ts.Fatalf("signal: %s", problem) + } + if code != wantCode { + ts.Fatalf("signal: %s returned %s, want %s", args[0], code, wantCode) + } + same, err := unchanged(protected, before, marker) + ts.Check(err) + if !same { + ts.Fatalf("signal: %s changed protected paths: %s", args[0], fmt.Sprint(protected)) + } +} + +// foreground runs a resolved call in its own process group, with the scenario's stdin, environment, and folder. +// act runs beside the host until it returns. foreground returns the exit and both streams of the host. +func foreground(ts *testscript.TestScript, name, program string, args []string, act func(done <-chan struct{}, host *os.Process) error) (int, string, string) { + dir, err := os.MkdirTemp("", "ncly-"+name+"-") ts.Check(err) defer func() { _ = os.RemoveAll(dir) }() // Testscript exposes neither stdin nor the complete scenario environment to custom commands. // Capture both before starting a command whose process group and stream drain the kit controls. envfile, inputfile := filepath.Join(dir, "env"), filepath.Join(dir, "stdin") - ts.Check(ts.Exec("/bin/sh", "-c", `/usr/bin/env -0 > "$1" && /bin/cat > "$2"`, "signal", envfile, inputfile)) + ts.Check(ts.Exec("/bin/sh", "-c", `/usr/bin/env -0 > "$1" && /bin/cat > "$2"`, name, envfile, inputfile)) env, err := os.ReadFile(envfile) ts.Check(err) input, err := os.Open(inputfile) ts.Check(err) defer func() { _ = input.Close() }() - cmd := exec.Command(program, args[3:]...) + cmd := exec.Command(program, args...) cmd.Dir = ts.MkAbs(".") cmd.Env = strings.Split(strings.TrimSuffix(string(env), "\x00"), "\x00") cmd.Stdin = input @@ -48,64 +77,81 @@ func (c *Coverage) signal(ts *testscript.TestScript, neg bool, args, survives [] cmd.WaitDelay = runner.DrainLimit ts.Check(cmd.Start()) defer func() { _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }() - sig, wantExit, wantCode := syscall.SIGINT, 130, contract.Interrupted - if args[0] == "TERM" { - sig, wantExit, wantCode = syscall.SIGTERM, 143, contract.Terminated - } done, result := make(chan struct{}), make(chan error, 1) - go func() { result <- sendSignal(done, cmd.Process, marker, survives, sig) }() + go func() { result <- act(done, cmd.Process) }() exit, runErr := exitCode(cmd.Wait()) close(done) _, _ = ts.Stdout().Write(stdout.Bytes()) _, _ = ts.Stderr().Write(stderr.Bytes()) ts.Check(<-result) ts.Check(runErr) - code, problem := c.checkedCode(words, exit, stdout.String(), stderr.String()) - if exit != wantExit { - ts.Fatalf("signal: %s exited with %d, want %d", args[2], exit, wantExit) - } - if problem != "" { - ts.Fatalf("signal: %s", problem) - } - if code != wantCode { - ts.Fatalf("signal: %s returned %s, want %s", args[0], code, wantCode) - } + return exit, stdout.String(), stderr.String() } -// sendSignal waits for the host's new marker, captures finished files, and bounds both waits. -func sendSignal(done <-chan struct{}, host *os.Process, marker string, survives []string, sig syscall.Signal) error { +// waitMarker waits up to 10 seconds for the host's new regular marker file. +func waitMarker(name string, done <-chan struct{}, host *os.Process, marker string) error { deadline := time.NewTimer(10 * time.Second) defer deadline.Stop() tick := time.NewTicker(10 * time.Millisecond) defer tick.Stop() - kill := func() { _ = syscall.Kill(-host.Pid, syscall.SIGKILL) } for { if info, err := os.Lstat(marker); err == nil && info.Mode().IsRegular() { - break + return nil } select { case <-done: - return fmt.Errorf("signal: host exited before marker %s", marker) + return fmt.Errorf("%s: host exited before marker %s", name, marker) case <-deadline.C: - kill() - return fmt.Errorf("signal: timed out waiting for marker %s", marker) + kill(host) + return fmt.Errorf("%s: timed out waiting for marker %s", name, marker) case <-tick.C: } } +} + +// waitExit waits up to limit for the host to exit, and kills its process group after that. +func waitExit(done <-chan struct{}, host *os.Process, limit time.Duration) bool { + deadline := time.NewTimer(limit) + defer deadline.Stop() + select { + case <-done: + return true + case <-deadline.C: + kill(host) + return false + } +} + +func kill(host *os.Process) { _ = syscall.Kill(-host.Pid, syscall.SIGKILL) } + +// unchanged compares protected paths with their state before the call. The kit's own files are skipped. +func unchanged(protected []string, before map[string]fileState, skip ...string) (bool, error) { + after, err := snapshotFiles(protected, false) + if err != nil { + return false, err + } + for _, path := range skip { + delete(before, path) + delete(after, path) + } + return maps.Equal(before, after), nil +} + +// sendSignal waits for the host's new marker, captures finished files, and bounds both waits. +func sendSignal(done <-chan struct{}, host *os.Process, marker string, survives []string, sig syscall.Signal) error { + if err := waitMarker("signal", done, host, marker); err != nil { + return err + } before, err := snapshotFiles(survives, true) if err != nil { - kill() + kill(host) return err } if err := host.Signal(sig); err != nil { - kill() + kill(host) return fmt.Errorf("signal: send %s to host: %w", sig, err) } - deadline.Reset(runner.Cleanup + runner.DrainLimit + 3*time.Second) - select { - case <-done: - case <-deadline.C: - kill() + if !waitExit(done, host, runner.Cleanup+runner.DrainLimit+3*time.Second) { return fmt.Errorf("signal: host did not exit after %s", sig) } after, err := snapshotFiles(survives, true) From 005e001d4a81dbd8ad4ffb0c73417f052552a0ee Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:45:19 -0400 Subject: [PATCH 15/61] =?UTF-8?q?=E2=9C=A8=20feat:=20sdk/extensiontest:=20?= =?UTF-8?q?appears=20proves=20that=20a=20run=20never=20replaces=20a=20new?= =?UTF-8?q?=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit appears ... runs a registered call as signal does. At the new marker, the kit creates the path with an exclusive create and its own bytes. The run must then fail with a valid answer, and the path must stay the same regular file with the same bytes. The command refuses --force, and the next protect selection applies to it. Scenario commands lists which kit calls prove each invariant of a command that replaces a user file. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/scenario-commands.md | 30 ++++- docs/north-star/contract/test-kit.md | 4 +- sdk/extensiontest/appears.go | 104 ++++++++++++++++++ sdk/extensiontest/appears_test.go | 64 +++++++++++ sdk/extensiontest/extensiontest.go | 3 +- sdk/extensiontest/paired.go | 18 ++- 6 files changed, 209 insertions(+), 14 deletions(-) create mode 100644 sdk/extensiontest/appears.go create mode 100644 sdk/extensiontest/appears_test.go diff --git a/docs/north-star/contract/scenario-commands.md b/docs/north-star/contract/scenario-commands.md index 4338dc1a..fe83c412 100644 --- a/docs/north-star/contract/scenario-commands.md +++ b/docs/north-star/contract/scenario-commands.md @@ -12,17 +12,18 @@ A scenario uses the commands of [testscript](https://pkg.go.dev/github.com/rogpe | `exits ...` | Runs a command, these two included, and checks its exact exit code | | `signal INT\|TERM ...` | Waits for a new marker, signals the host, and checks its exit and answer | | `survives ...` | Requires these finished files at the next signal's marker and unchanged after its exit | -| `protect ...` | Selects files, folders, or absent paths that the next `signal` call and subsequent `refuses` calls must keep unchanged | +| `appears ...` | Creates a new file at the marker, and requires a failed run that keeps that file | +| `protect ...` | Selects files, folders, or absent paths that the next `signal` or `appears` call and subsequent `refuses` calls must keep unchanged | | `refuses ...` | Requires the same refusal code and exit in a dry run and a run | | `grant ` | Grants the key of a declared service to the extension at its scratch path | | `fake ` | Puts the fake child in `$WORK/bin`, first in `PATH`, under the name `name` | The fake child runs the actions of `$WORK/bin/.fake`, one per line: `stdout `, `stderr `, `stdin`, `env`, `sleep `, and `exit `. `env` writes the names of its variables, never a value. The package comment of `sdk/extensiontest/fake` defines each action. -`signal` and `refuses` accept only registered `ncly` and `ncly-` calls of runnable commands. +`signal`, `appears`, and `refuses` accept only registered `ncly` and `ncly-` calls of runnable commands. They enable JSON and non-interactive mode, and validate each answer through `Coverage.Observe`. Explicit flags that disable JSON or non-interactive mode are refused. -Both commands leave the last call's streams available to `stdout` and `stderr` assertions. +These commands leave the last call's streams available to `stdout` and `stderr` assertions. Their `ncly` calls use the registered host's absolute path, even when a scenario replaces `PATH`. `signal` runs the host in the foreground, without a shell cleanup command supplying its result. @@ -45,11 +46,30 @@ Nicely's cache stays unprotected unless a scenario selects its path. Select inputs and possible outputs with `protect` before each refusal group. A `protect` selection applies to each later `refuses` call, until the next `protect` call replaces it. -It also applies to the next `signal` call, and that call ends it for `signal`. -Give a new `protect` call before each `signal` call that must keep paths. +It also applies to the next `signal` or `appears` call, which ends it for these two commands. +Give a new `protect` call before each `signal` or `appears` call that must keep paths. `signal` compares the protected paths before the host starts and after it exits, except its marker. So an interrupted run must keep its input, leave an absent output absent, and keep an old output. To prove that a command never writes through a link, make its output a symbolic link to its input. Protect both paths, then call `refuses`. A write through the link changes the input, and a replacement of the link changes the link. + +`appears` proves that a run without `--force` never replaces a file that appears during the run. +It refuses `--force` among the arguments. +Its marker and its path must not exist before the call. +It runs the host as `signal` does, and waits up to 10 seconds for a regular marker file. +Then it creates the path with an exclusive create, and writes bytes that the kit chooses. +So the host must wait for the path after it writes the marker, for example in a stub of the wrapped program. +After the path appears, the host has 10 seconds to exit, and `runner.DrainLimit` to close its streams. +The host must fail with a valid answer, and its exit must not be 0. +The path must stay the same regular file, with the same bytes and permissions. +So a run that renames over the file, writes through it, or puts a link in its place fails. +The protected paths must stay unchanged, except the marker and the path. + +These calls prove the invariants of a command that replaces a user file: + +- The input stays unchanged: `protect` the input before `refuses`, `signal`, and `appears`. +- A failure leaves no partial output and keeps the old output: `protect` the output path or its folder before `refuses` and `signal`. +- The command never writes through a link: `refuses`, with the output as a symbolic link to the input. +- Without `--force`, the command never replaces a file that appears during the run: `appears`. diff --git a/docs/north-star/contract/test-kit.md b/docs/north-star/contract/test-kit.md index 288a7c07..f12ab355 100644 --- a/docs/north-star/contract/test-kit.md +++ b/docs/north-star/contract/test-kit.md @@ -29,7 +29,7 @@ A scratch `HOME` does not hide the OS keychain. `ncly` reads it only when the va - `extension.Check`, and a skill named after the slug, which every external extension ships, as [Extensions](extensions.md) defines - Each answer of a call through `ncly` or `ncly-`, for a success and for a failure. Its schema is the one that `contract.AnswerSchema` builds from the command's declaration - No scenario runs `ncly` or `ncly-` through `exec` in the foreground, by name or by path, which would skip the answer check. The kit splits each line into words as testscript does, so quotes and comments hide no call. A background call, which ends with `&`, keeps `exec`, and the kit does not check its answer -- Each runnable command has a recorded call through `ncly`, `ncly-`, `exits`, `signal`, or `refuses` after its answer passes validation +- Each runnable command has a recorded call through `ncly`, `ncly-`, `exits`, `signal`, `appears`, or `refuses` after its answer passes validation - A command whose declared result kind is `text` counts a completed success, since that kind has no JSON answer - Coverage runs after all scenarios finish. Comments, assertions, skipped conditions, help, and shell or background `exec` calls earn no coverage - Human text earns no coverage for a command with a structured result @@ -52,7 +52,7 @@ Nicely's runner registers these checks for `agent` and `transcript`, for success It checks declared codes for `hello`, whose signal commands run in the background. Its path-qualified direct calls pass the same answer check. Its `exits` command checks successful and failed calls, including `exits 0` for successes. -`Coverage.ScenarioCommands` takes the registered host's absolute path and supplies `signal`, `survives`, `protect`, and `refuses` to other scenario runners. +`Coverage.ScenarioCommands` takes the registered host's absolute path and supplies `signal`, `survives`, `appears`, `protect`, and `refuses` to other scenario runners. ## Manifest examples diff --git a/sdk/extensiontest/appears.go b/sdk/extensiontest/appears.go new file mode 100644 index 00000000..13f38ee6 --- /dev/null +++ b/sdk/extensiontest/appears.go @@ -0,0 +1,104 @@ +package extensiontest + +import ( + "fmt" + "os" + "slices" + "strings" + "time" + + "github.com/pascalandy/nicely/sdk/runner" + "github.com/rogpeppe/go-internal/testscript" +) + +// appearsText is what the kit writes at the path. A run that replaces the file cannot keep its inode. +const appearsText = "Another process wrote this file during the run. The run must keep it.\n" + +// appears proves that a run without --force never replaces a file that appears during the run. +// At the marker, the kit creates the path with an exclusive create. The run must then fail and keep that file. +func (c *Coverage) appears(ts *testscript.TestScript, neg bool, args, protected []string, registeredHost string) { + if neg || len(args) < 3 { + ts.Fatalf("usage: appears ...") + } + defer machineMode(ts)() + program, words, call := c.pairedCall(ts, args[2], args[3:], registeredHost) + if _, given := call.line.Switch("force"); given || slices.ContainsFunc(args[3:], func(arg string) bool { + return arg == "--force" || strings.HasPrefix(arg, "--force=") + }) { + ts.Fatalf("appears: remove --force, because appears proves a run without it") + } + marker, path := ts.MkAbs(args[0]), ts.MkAbs(args[1]) + for i, file := range []string{marker, path} { + if _, err := os.Lstat(file); !os.IsNotExist(err) { + ts.Fatalf("appears: %s must not exist before the call", args[i]) + } + } + before, err := snapshotFiles(protected, false) + ts.Check(err) + var made os.FileInfo + exit, stdout, stderr := foreground(ts, "appears", program, args[3:], func(done <-chan struct{}, host *os.Process) error { + if err := waitMarker("appears", done, host, marker); err != nil { + return err + } + info, err := createNew(path) + if err != nil { + kill(host) + return fmt.Errorf("appears: %s exists at the marker, or the kit cannot create it: %w", args[1], err) + } + made = info + if !waitExit(done, host, 10*time.Second+runner.DrainLimit) { + return fmt.Errorf("appears: host did not exit within 10s after %s appeared", args[1]) + } + return nil + }) + if _, problem := c.checkedCode(words, exit, stdout, stderr); problem != "" { + ts.Fatalf("appears: %s", problem) + } + if exit == 0 { + ts.Fatalf("appears: %s exited with 0, want a failure that keeps %s", args[2], args[1]) + } + ts.Check(kept(args[1], path, made)) + same, err := unchanged(protected, before, marker, path) + ts.Check(err) + if !same { + ts.Fatalf("appears: the run changed protected paths: %s", fmt.Sprint(protected)) + } +} + +// createNew creates path with an exclusive create and writes appearsText. +// It records the file before the write, because the host can act as soon as the path exists. +func createNew(path string) (os.FileInfo, error) { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644) + if err != nil { + return nil, err + } + info, err := file.Stat() + if err == nil { + _, err = file.WriteString(appearsText) + } + if closeErr := file.Close(); err == nil { + err = closeErr + } + return info, err +} + +// kept requires the same regular file at path, with the same mode and bytes. +func kept(name, path string, made os.FileInfo) error { + info, err := os.Lstat(path) + switch { + case err != nil: + return fmt.Errorf("appears: the run removed %s: %w", name, err) + case !info.Mode().IsRegular(): + return fmt.Errorf("appears: %s is no longer the regular file that the kit created", name) + case !os.SameFile(made, info): + return fmt.Errorf("appears: the run replaced %s", name) + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + if string(data) != appearsText || info.Mode() != made.Mode() { + return fmt.Errorf("appears: the run changed %s", name) + } + return nil +} diff --git a/sdk/extensiontest/appears_test.go b/sdk/extensiontest/appears_test.go new file mode 100644 index 00000000..f37889b2 --- /dev/null +++ b/sdk/extensiontest/appears_test.go @@ -0,0 +1,64 @@ +package extensiontest_test + +import ( + "strings" + "testing" +) + +// appearsHost copies its input to a temporary file and writes the marker. +// It waits for the output to appear, then runs commit. +func appearsHost(beforeMarker, commit string) string { + return `#!/bin/sh +[ "$1" = copy ] && shift +input=$1 output=$2 tmp=$(dirname "$2")/.copy.$$ +cat "$input" > "$tmp" +` + beforeMarker + `echo ready > started +i=0 +until [ -e "$output" ] || [ "$i" -ge 500 ]; do i=$((i+1)); sleep 0.01; done +` + commit +} + +// safeCommit links the temporary file to the output, which fails when the output exists. +const safeCommit = `if ln "$tmp" "$output" 2>/dev/null; then + rm -f "$tmp" + echo '{"ok":true,"contract_version":1,"output":"'"$output"'"}' + exit 0 +fi +rm -f "$tmp" +` + copyRefusal + +// TestAppears proves that appears accepts a run that keeps a new file, and rejects each unsafe run. +func TestAppears(t *testing.T) { + safe := appearsHost("", safeCommit) + call := "appears started out/notes.txt ncly-standalone copy notes.md out/notes.txt\n" + cases := []struct{ name, host, script, problem string }{ + {"keeps the new file", safe, "protect notes.md out\n" + call + "stderr '\"code\":\"USAGE_INVALID\"'\n", ""}, + {"through ncly", safe, "appears started out/notes.txt ncly standalone copy notes.md out/notes.txt\n", ""}, + {"rename over the new file", appearsHost("", "mv -f \"$tmp\" \"$output\"\n"+copyRefusal), call, "the run replaced out/notes.txt"}, + {"rename with the same bytes", appearsHost("", "cat \"$output\" > \"$tmp\"\nmv -f \"$tmp\" \"$output\"\n"+copyRefusal), call, "the run replaced out/notes.txt"}, + {"write through the new file", appearsHost("", "cat \"$tmp\" > \"$output\"\nrm -f \"$tmp\"\n"+copyRefusal), call, "the run changed out/notes.txt"}, + {"link replaces the new file", appearsHost("", "rm -f \"$output\"\nln -s ../notes.md \"$output\"\nrm -f \"$tmp\"\n"+copyRefusal), call, "out/notes.txt is no longer the regular file"}, + {"success over the new file", appearsHost("", "rm -f \"$tmp\"\necho '{\"ok\":true,\"contract_version\":1,\"output\":\"out/notes.txt\"}'\nexit 0\n"), call, "exited with 0, want a failure"}, + {"temporary file left", appearsHost("", copyRefusal), "protect notes.md out\n" + call, "changed protected paths"}, + {"input changed", appearsHost("", "rm -f \"$tmp\"\necho more >> \"$input\"\n"+copyRefusal), "protect notes.md\n" + call, "changed protected paths"}, + {"output before the marker", appearsHost("cp \"$tmp\" \"$output\"\n", "sleep 1\n"+safeCommit), call, "exists at the marker"}, + {"output before the call", safe, "cp notes.md out/notes.txt\n" + call, "out/notes.txt must not exist before the call"}, + {"stale marker", safe, "cp notes.md started\n" + call, "started must not exist before the call"}, + {"exit before the marker", "#!/bin/sh\n" + copyRefusal, call, "host exited before marker"}, + {"force", safe, "appears started out/notes.txt ncly-standalone copy notes.md out/notes.txt --force\n", "remove --force"}, + {"answer schema", appearsHost("", "rm -f \"$tmp\"\n"+strings.Replace(copyRefusal, `"ok":false`, `"ok":"false"`, 1)), call, "breaks the schema"}, + {"path bypass", safe, "appears started out/notes.txt $XDG_DATA_HOME/nicely/extensions/ncly-standalone copy notes.md out/notes.txt\n", "use the registered command"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + out, err := copyFixture(t, c.host, c.script) + if c.problem == "" { + if err != nil { + t.Fatalf("valid appears scenario failed: %v\n%s", err, out) + } + } else if err == nil || !strings.Contains(out, c.problem) { + t.Fatalf("want %q, got %v\n%s", c.problem, err, out) + } + }) + } +} diff --git a/sdk/extensiontest/extensiontest.go b/sdk/extensiontest/extensiontest.go index 4e0d644e..b7c8288f 100644 --- a/sdk/extensiontest/extensiontest.go +++ b/sdk/extensiontest/extensiontest.go @@ -14,7 +14,8 @@ // exits runs a command, these two included, and checks its exact exit code // signal INT|TERM checks the host's signal exit and answer // survives ... requires finished files before and after the next signal -// protect ... selects paths that the next signal and later refusal pairs must preserve +// appears requires a failure that keeps a new file +// protect ... selects paths that refuses and the next signal or appears keep // refuses checks the same refusal in a dry run and a run // grant grants the key of a declared service to the extension // fake puts the fake child in $WORK/bin as name diff --git a/sdk/extensiontest/paired.go b/sdk/extensiontest/paired.go index 85042510..c5a63008 100644 --- a/sdk/extensiontest/paired.go +++ b/sdk/extensiontest/paired.go @@ -15,12 +15,12 @@ import ( ) // scriptFiles holds the selections of one scenario. -// protect lasts until the next protect call, and next until the next signal call. +// protect lasts until the next protect call, and next until the next signal or appears call. type scriptFiles struct { protect, next, survives []string } -// ScenarioCommands supplies signal, refuses, protect, and survives to a testscript runner. +// ScenarioCommands supplies signal, appears, refuses, protect, and survives to a testscript runner. // host is the registered ncly's absolute path, independent of the scenario's PATH. func (c *Coverage) ScenarioCommands(host string) map[string]func(*testscript.TestScript, bool, []string) { var mu sync.Mutex @@ -63,6 +63,12 @@ func (c *Coverage) ScenarioCommands(host string) map[string]func(*testscript.Tes defer func() { files.survives = nil }() c.signal(ts, neg, args, files.survives, next, host) }, + "appears": func(ts *testscript.TestScript, neg bool, args []string) { + files := state(ts) + next := files.next + files.next = nil + c.appears(ts, neg, args, next, host) + }, "refuses": func(ts *testscript.TestScript, neg bool, args []string) { c.refuses(ts, neg, args, state(ts).protect, host) }, @@ -89,16 +95,16 @@ func (c *Coverage) pairedCall(ts *testscript.TestScript, program string, args [] ts.Fatalf("%s: expected a runnable command of %s", program, c.manifest.Slug()) } if on, given := call.line.Switch("json"); given && !on { - ts.Fatalf("%s: signal and refuses require JSON", program) + ts.Fatalf("%s: signal, appears, and refuses require JSON", program) } if on, given := call.line.Switch("no-input"); given && !on { - ts.Fatalf("%s: signal and refuses require non-interactive mode", program) + ts.Fatalf("%s: signal, appears, and refuses require non-interactive mode", program) } if on, _ := call.line.Switch("help"); on { - ts.Fatalf("%s: signal and refuses cannot check help", program) + ts.Fatalf("%s: signal, appears, and refuses cannot check help", program) } if on, _ := call.line.Switch("version"); on { - ts.Fatalf("%s: signal and refuses cannot check version", program) + ts.Fatalf("%s: signal, appears, and refuses cannot check version", program) } return path, words, call } From 2c9fb2128a7ef42826c8d9c0a9032e2cd697f3d3 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 14:45:38 -0400 Subject: [PATCH 16/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20contract:=20accept?= =?UTF-8?q?ance.sh=20checks=20the=20output=20with=20an=20independent=20rea?= =?UTF-8?q?der?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A check that reads only a header or the file type cannot see a lost content. The check now compares what a reader other than the wrapped program gives with an independent reference. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/test-kit.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/north-star/contract/test-kit.md b/docs/north-star/contract/test-kit.md index f12ab355..3d6da4eb 100644 --- a/docs/north-star/contract/test-kit.md +++ b/docs/north-star/contract/test-kit.md @@ -46,6 +46,10 @@ The fixture extension repository in `testdata/extensions/standalone/` proves eac The kit proves the declared cases. Whether a claimed test proves its line, and whether a command has a hidden effect, stay judgments for the author and the reviewer. +`acceptance.sh` checks the primary output with a reader that is not the wrapped program. +A check that reads only a header or the file type is not sufficient. +The check compares the content that this reader gives with an independent reference, such as a file of expected text that the author writes. + Other scenario runners use `NewCoverage`, `Coverage.Observe`, and `Coverage.Missing` for the same answer and coverage checks. `Observe` receives completed calls, including their exit code and both streams, without changing process execution or output. Nicely's runner registers these checks for `agent` and `transcript`, for successful and failed calls. From fd00b34cfe3641b80135132dcd4c9259b77390a3 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 15:02:04 -0400 Subject: [PATCH 17/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20template:=20the=20?= =?UTF-8?q?skill=20names=20the=20kit=20commands=20that=20prove=20a=20comma?= =?UTF-8?q?nd=20that=20replaces=20a=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- template/.agents/skills/nicely-extension/SKILL.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index be0dcb85..968c6c82 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -128,13 +128,16 @@ The release receipt and the human items of `qa.md` come after the merge, as "Rel ### Commands that replace a file -For a command that replaces a user file, prove these invariants with the scenario commands of `test-kit.md` in the contract: +For a command that replaces a user file, prove these invariants with the commands of `scenario-commands.md` in the contract: -- The input stays unchanged: `protect` the input before each `refuses` and `signal` call -- A failure leaves no partial output and keeps the old output: `protect` the output path, absent or old, before each `refuses` and `signal` call +- The input stays unchanged: `protect` the input before each `refuses`, `signal`, and `appears` call +- A failure leaves no partial output and keeps the old output: `protect` the output path, absent or old, before each of these calls - The command never writes through a link: give an output that is a link to the input, `protect` both, and refuse the call with `refuses` - Without `--force`, the command never replaces a file that appears during the run: `appears` creates that file, and the call keeps it +A `protect` before a `signal` or an `appears` call covers that one call. +For `appears`, the stub of the wrapped program writes the marker, then waits until the file exists. + For a command that replays paid work, run `create-formal-verification-skill`. ## Readiness review From 5859d6934496dffad9d8f92082a392741edba4f0 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 15:02:19 -0400 Subject: [PATCH 18/61] =?UTF-8?q?=E2=9C=85=20test:=20template:=20the=20pro?= =?UTF-8?q?of=20of=20finish=20names=20an=20author=20in=20place=20of=20a=20?= =?UTF-8?q?place?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- scripts/template-finish-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/template-finish-test.sh b/scripts/template-finish-test.sh index 20f29e85..fdbf32ef 100644 --- a/scripts/template-finish-test.sh +++ b/scripts/template-finish-test.sh @@ -14,7 +14,7 @@ fct_test_finish() { local unfilled='this line of the form is not filled' local warn=$'### R1-F1 Large files are slow\n\n- **Severity**: `warn`' local advise=$'### R1-F2 The count could name the longest line\n\n- **Severity**: `advise`' - local hidden=$'### R1-F2 The count could name the longest line\n\n- **Severity**: `advise`\n\n| Output keeps GPS | `warn` |\n| Output keeps the serial number | block |\n\n```\n### Output keeps the place\n\n- **Severity**: `warn`\n```\n\n### R1-F3 Output keeps the name\n\n- Severity: warn\n- **Severity**: `advise`\n\n### R1-F4 Output keeps the date\n\nSeverity: warn.\n\n- **Severity**: `advise`\n\n### Severity: warn\n\n- **Severity**: `advise`\n- **Spec**: `block`' + local hidden=$'### R1-F2 The count could name the longest line\n\n- **Severity**: `advise`\n\n| Output keeps the author | `warn` |\n| Output keeps the serial number | block |\n\n```\n### Output keeps the place\n\n- **Severity**: `warn`\n```\n\n### R1-F3 Output keeps the name\n\n- Severity: warn\n- **Severity**: `advise`\n\n### R1-F4 Output keeps the date\n\nSeverity: warn.\n\n- **Severity**: `advise`\n\n### Severity: warn\n\n- **Severity**: `advise`\n- **Spec**: `block`' local unread=$'- **Severity**: `warn`\n\n### No severity\n\n### Two severities\n\n- **Severity**: `advise`\n- **Severity**: `warn`\n\n### Unknown severity\n\n- **Severity**: minor\n\n### A fenced heading\n\n````\n# a comment\n```\n````\n\n- **Severity**: `block`\n\n### An open fence\n\n```\n- **Severity**: `advise`' cp -R "${1}" "${finish}" [[ "$(just --justfile "${finish}/justfile" --show finish | sed '/^#/d')" == $'finish:\n go run github.com/pascalandy/nicely/sdk/extensionrepo/cmd/extensionrepo finish' ]] || From 8e5da038f08c3585051230fa2c34cb8d24374e43 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 15:19:11 -0400 Subject: [PATCH 19/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20a=20mov?= =?UTF-8?q?ed=20intake=20or=20a=20hidden=20row=20never=20resets=20the=20le?= =?UTF-8?q?dger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 1 found that a readiness.md could move its Intake to an earlier review, drop a warn, and restart the rounds at 1. A label line under ## Rounds could also hide rows. The history of a build now starts only after a committed review with a passing verdict, so a review that needs work never ends a build. Each round of an earlier committed ledger must stay unchanged. Each line under ## Rounds and ## Order of round 1 must be a row. Each of the first four steps of round 1 needs its own commit. The rule index lists appears among the call forms. The no-python line of the template spec names the probe that does not finish within 10 s again, and a hello scenario proves it. Part of #417. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 10 ++- scripts/template-finish-helpers.sh | 41 ++++++--- scripts/template-finish-test.sh | 6 +- scripts/template-ledger-test.sh | 86 ++++++++++++------ scripts/template-signoff-test.sh | 1 + sdk/extensionrepo/ledger.go | 89 ++++++++++++++----- sdk/extensionrepo/ledger_findings.go | 32 +++++-- .../nicely-extension/readiness-review.md | 2 + template/AGENTS/rule-index.md.jinja | 2 +- template/spec.md.jinja | 2 +- .../hello/testdata/script/wrap_stop.txtar | 9 ++ 11 files changed, 205 insertions(+), 75 deletions(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index 2f65d5c6..8c7271a4 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -47,13 +47,19 @@ A finding of the last round can have no state yet. `just finish` fails on each of these: - No round under `## Rounds`, or a row without its six cells, its number, a full Head, or both reviewers +- A line under `## Rounds` or `## Order of round 1` that is no row of its table, such as a label that would hide rows - More than 5 rounds. The message tells the agent to stop and to report to the user - A commit of a round or of a fixed finding that is not HEAD or an ancestor of HEAD - A last round whose Head is not the `Commit:` of `readiness.md` -- A step of round 1 that is missing or out of order, or a handler in the commit of the first scenario +- A step of round 1 that is missing, out of order, or in the commit of the step before it - A finding without an ID, a finding of an earlier round without a state, or a state without its parts - A finding with the state `open`, even when `## Open lines` holds `None.` - A `warn` or `block` finding with the state `declined` -- A finding of an earlier committed `readiness.md` that is missing, or whose severity went down. Finish reads the commits after the Intake commit of the build, so an earlier build is no history of this one +- A finding of an earlier committed `readiness.md` that is missing, or whose severity went down +- A round of an earlier committed ledger that is missing or changed. Only a Finish that was `pending` changes + +The history of a build starts after the last committed `readiness.md` with a passing verdict at or before its Intake commit. +So an earlier build that ended is no history of this one. +A review that needs work never ends a build, so a moved Intake hides no finding and resets no round. At 5 rounds, a failing `just finish` also tells the agent to stop and to report each open line and each open finding. diff --git a/scripts/template-finish-helpers.sh b/scripts/template-finish-helpers.sh index 004033c7..c5088bf4 100644 --- a/scripts/template-finish-helpers.sh +++ b/scripts/template-finish-helpers.sh @@ -25,8 +25,8 @@ fct_finish_said() { # fct_readiness writes readiness.md for the commit $1, with the verdict $2, the # findings $3 under ## Findings, and the reviewer $4. An empty $3 leaves out -# ## Findings, and an empty $4 leaves out the line Reviewer:. The ledger -# holds $5 rounds, or 1, from the intake $6, and its last round reviewed $1. +# ## Findings, and an empty $4 leaves out the line Reviewer:. The rounds of the +# ledger reviewed the heads of ledger_earlier, then $1. fct_readiness() { local reviewer="${4-an agent that wrote none of the code}" { @@ -34,22 +34,37 @@ fct_readiness() { [[ -z "${reviewer}" ]] || printf 'Reviewer: %s\n' "${reviewer}" printf 'Verdict: %s\n\n# Readiness review\n' "${2}" [[ -z "${3}" ]] || printf '\n## Findings\n\n%s\n' "${3}" - fct_ledger "${finish}" "${1}" "${5:-1}" "${6:-}" + fct_ledger "${finish}" "${ledger_earlier:-} ${1}" } >"${finish}/readiness.md" } -# fct_ledger prints the ledger of a build of $3 rounds in the repository $1, -# whose last round reviewed $2. The build starts at the intake $4, or at the -# first commit. Earlier rounds name that intake, and round 1 commits its -# handler in $2 (#417). +# fct_steps commits the first three steps of round 1 in the repository $1, +# as empty commits, so the ledger can name their order (#417). +fct_steps() { + local step + for step in intake spec scenario; do + git -C "${1}" commit -q --allow-empty -m "step: ${step}" + done +} + +# fct_step prints the newest commit of the step $2 in the repository $1. +fct_step() { + git -C "${1}" log --format=%H -n 1 -F --grep="step: ${2}" HEAD +} + +# fct_ledger prints the ledger of the repository $1 for the rounds that +# reviewed the heads $2, in order. The newest commits of fct_steps give the +# order of round 1, and its handler is in the first head (#417). fct_ledger() { - local first="${4}" i - [[ -n "${first}" ]] || first="$(git -C "${1}" rev-list --max-parents=0 HEAD)" + local heads intake i finish_cell + read -r -a heads <<<"${2}" + intake="$(fct_step "${1}" intake)" printf '\n## Rounds\n\n| Round | Head | Change review | Change commits | Readiness review | Finish |\n|---|---|---|---|---|---|\n' - for ((i = 1; i < ${3}; i++)); do - printf '| %s | %s | agent A | %s..%s | agent B | `fail` |\n' "${i}" "${first}" "${first}" "${first}" + for ((i = 0; i < ${#heads[@]}; i++)); do + finish_cell='`fail`' + [[ "$((i + 1))" != "${#heads[@]}" ]] || finish_cell='`pending`' + printf '| %s | %s | agent A | %s..%s | agent B | %s |\n' "$((i + 1))" "${heads[i]}" "${intake}" "${heads[i]}" "${finish_cell}" done - printf '| %s | %s | agent A | %s..%s | agent B | `pending` |\n' "${3}" "${2}" "${first}" "${2}" printf '\n## Order of round 1\n\n| Step | Commit |\n|---|---|\n' - printf '| %s | %s |\n' Intake "${first}" Spec "${first}" Scenario "${first}" Handler "${2}" + printf '| %s | %s |\n' Intake "${intake}" Spec "$(fct_step "${1}" spec)" Scenario "$(fct_step "${1}" scenario)" Handler "${heads[0]:-}" } diff --git a/scripts/template-finish-test.sh b/scripts/template-finish-test.sh index fdbf32ef..dab71a5b 100644 --- a/scripts/template-finish-test.sh +++ b/scripts/template-finish-test.sh @@ -9,7 +9,7 @@ source "${BASH_SOURCE[0]%/*}/template-finish-helpers.sh" fct_test_finish() { - local finish="${2}/finish" scratch="${2}" sha file number + local finish="${2}/finish" scratch="${2}" sha file number ledger_earlier="" local free='this limit has no spec ID, no state limit, and no acceptance' local unfilled='this line of the form is not filled' local warn=$'### R1-F1 Large files are slow\n\n- **Severity**: `warn`' @@ -17,6 +17,7 @@ fct_test_finish() { local hidden=$'### R1-F2 The count could name the longest line\n\n- **Severity**: `advise`\n\n| Output keeps the author | `warn` |\n| Output keeps the serial number | block |\n\n```\n### Output keeps the place\n\n- **Severity**: `warn`\n```\n\n### R1-F3 Output keeps the name\n\n- Severity: warn\n- **Severity**: `advise`\n\n### R1-F4 Output keeps the date\n\nSeverity: warn.\n\n- **Severity**: `advise`\n\n### Severity: warn\n\n- **Severity**: `advise`\n- **Spec**: `block`' local unread=$'- **Severity**: `warn`\n\n### No severity\n\n### Two severities\n\n- **Severity**: `advise`\n- **Severity**: `warn`\n\n### Unknown severity\n\n- **Severity**: minor\n\n### A fenced heading\n\n````\n# a comment\n```\n````\n\n- **Severity**: `block`\n\n### An open fence\n\n```\n- **Severity**: `advise`' cp -R "${1}" "${finish}" + fct_steps "${finish}" [[ "$(just --justfile "${finish}/justfile" --show finish | sed '/^#/d')" == $'finish:\n go run github.com/pascalandy/nicely/sdk/extensionrepo/cmd/extensionrepo finish' ]] || fct_fail 'just finish must call the SDK program in one line' @@ -61,6 +62,7 @@ fct_test_finish() { fct_readiness "${sha}" Ready "${advise}" git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'independent readiness review' + ledger_earlier="${sha}" fct_finish 0 fct_finish_said 'finish: no line or finding is open' @@ -133,7 +135,7 @@ fct_test_finish() { fct_finish 1 fct_finish_said 'readiness.md: its first three lines hold no line Commit: or Verdict: ' - fct_readiness "${sha}" 'Ready with stated limits' "${warn}"$'\n- **State**: `limit`\n- **Spec**: `large-file`\n\n'"${advise}"$'\n\n## Severity assessment\n\nEach finding names its severity.' + fct_readiness "${sha}" 'Ready with stated limits' "${warn}"$'\n- **State**: `limit`\n- **Spec**: `large-file`\n\n'"${advise}"$'\n- **State**: `declined`\n- **Reason**: the answer stays small\n\n## Severity assessment\n\nEach finding names its severity.' git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'readiness review' fct_finish 0 diff --git a/scripts/template-ledger-test.sh b/scripts/template-ledger-test.sh index ebeb78af..10d8801e 100644 --- a/scripts/template-ledger-test.sh +++ b/scripts/template-ledger-test.sh @@ -6,79 +6,111 @@ # shellcheck disable=SC2016 fct_test_ledger() { - local finish="${2}/ledger" scratch="${2}" file first round2 head stray valid + local finish="${2}/ledger" scratch="${2}" file intake round1 review1 round2 review2 head stray valid ledger_earlier="" local advise=$'### R1-F1 The count could name the longest line\n\n- **Severity**: `advise`' local declined=$'\n- **State**: `declined`\n- **Reason**: the answer stays small' local warn=$'### R2-F1 Long lines are cut\n\n- **Severity**: `warn`' local lowered=$'### R2-F1 Long lines are cut\n\n- **Severity**: `advise`' cp -R "${1}" "${finish}" + fct_steps "${finish}" + intake="$(fct_step "${finish}" intake)" for file in vision.md spec.md qa.md; do grep -v '^Fill:' "${finish}/${file}" >"${scratch}/form.md" mv "${scratch}/form.md" "${finish}/${file}" done git -C "${finish}" commit -qam 'fill the forms' - first="$(git -C "${finish}" rev-list --max-parents=0 HEAD)" + round1="$(git -C "${finish}" rev-parse HEAD)" # Round 1 reports an advise, and round 2 declines it and reports a warn. # Their committed reviews are the history of round 3. - fct_readiness "$(git -C "${finish}" rev-parse HEAD)" 'Needs work' "${advise}" + fct_readiness "${round1}" 'Needs work' "${advise}" git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'readiness review of round 1' + review1="$(git -C "${finish}" rev-parse HEAD)" ledger_earlier="${round1}" printf 'Round 2.\n' >>"${finish}/README.md" git -C "${finish}" commit -qam 'round 2' round2="$(git -C "${finish}" rev-parse HEAD)" - fct_readiness "${round2}" 'Needs work' "${advise}${declined}"$'\n\n'"${warn}" 'agent B' 2 + fct_readiness "${round2}" 'Needs work' "${advise}${declined}"$'\n\n'"${warn}" 'agent B' git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'readiness review of round 2' + review2="$(git -C "${finish}" rev-parse HEAD)" ledger_earlier="${round1} ${round2}" printf 'Round 3.\n' >>"${finish}/README.md" git -C "${finish}" commit -qam 'round 3' head="$(git -C "${finish}" rev-parse HEAD)" # A warn of round 2 that the ledger of round 3 does not record fails finish. - fct_readiness "${head}" Ready "${advise}${declined}" 'agent B' 3 + fct_readiness "${head}" Ready "${advise}${declined}" 'agent B' fct_finish 1 fct_finish_said 'readiness.md: the finding R2-F1 of the review in' # A finding keeps its highest severity, so a warn never closes as a declined advise. - fct_readiness "${head}" Ready "${advise}${declined}"$'\n\n'"${lowered}${declined}" 'agent B' 3 + fct_readiness "${head}" Ready "${advise}${declined}"$'\n\n'"${lowered}${declined}" 'agent B' fct_finish 1 fct_finish_said 'the finding R2-F1 had the severity warn' - fct_readiness "${head}" Ready "${advise}${declined}"$'\n\n'"${warn}${declined}" 'agent B' 3 + fct_readiness "${head}" Ready "${advise}${declined}"$'\n\n'"${warn}${declined}" 'agent B' fct_finish 1 fct_finish_said 'the finding R2-F1 has the severity warn, and only an advise can be declined' - fct_readiness "${head}" Ready "${advise}${declined}"$'\n\n'"${warn}" 'agent B' 3 + fct_readiness "${head}" Ready "${advise}${declined}"$'\n\n'"${warn}" 'agent B' fct_finish 1 fct_finish_said 'the finding R2-F1 of round 2 has no state' # None. under ## Open lines does not close a finding that the ledger records as open. - fct_readiness "${head}" 'Ready with stated limits' "${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `open`\n- **Spec**: `long-lines`' 'agent B' 3 + fct_readiness "${head}" 'Ready with stated limits' "${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `open`\n- **Spec**: `long-lines`' 'agent B' fct_finish 1 fct_finish_said 'the finding R2-F1 is open, even if ## Open lines holds None.' - fct_readiness "${head}" Ready "${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `fixed`\n- **Spec**: `long-lines`' 'agent B' 3 + fct_readiness "${head}" Ready "${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `fixed`\n- **Spec**: `long-lines`' 'agent B' fct_finish 1 fct_finish_said 'this fixed finding names no commit of HEAD' fct_finish_said 'this fixed finding names no line of Failures or Acceptance in spec.md' valid="${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `fixed`\n- **Commit**: '"${head}"$'\n- **Spec**: `bad-input`' - fct_readiness "${head}" Ready "${valid}" 'agent B' 3 + fct_readiness "${head}" Ready "${valid}" 'agent B' git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'readiness review of round 3' fct_finish 0 + # Moving the intake after a review that needs work hides no finding and resets no round. + ledger_earlier="" + fct_readiness "${head}" Ready "${advise}${declined}" 'agent B' + fct_edit "${finish}/readiness.md" "| Intake | ${intake} |" "| Intake | ${review1} |" + fct_edit "${finish}/readiness.md" "| Spec | $(fct_step "${finish}" spec) |" "| Spec | ${round2} |" + fct_edit "${finish}/readiness.md" "| Scenario | $(fct_step "${finish}" scenario) |" "| Scenario | ${review2} |" + ledger_earlier="${round1} ${round2}" + fct_finish 1 + fct_finish_said 'readiness.md: the finding R2-F1 of the review in' + fct_finish_said 'readiness.md: round 1 of the ledger in' + # A round never leaves the ledger, and no row hides behind a label. + ledger_earlier="${round2}" + fct_readiness "${head}" Ready "${valid}" 'agent B' + ledger_earlier="${round1} ${round2}" + fct_finish 1 + fct_finish_said 'readiness.md: round 1 of the ledger in' + fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |" "| 3 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |"$'\n**Earlier build.**\n'"| 4 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |" + fct_finish 1 + fct_finish_said 'this line of ## Rounds is no row of its table' + # Each of the first four steps of round 1 has its own commit. + fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_edit "${finish}/readiness.md" "| Spec | $(fct_step "${finish}" spec) |" "| Spec | ${intake} |" + fct_finish 1 + fct_finish_said 'the step Spec is in the commit of the step before it' + # A build has at most 5 rounds, and finish tells the agent to stop. - fct_readiness "${head}" Ready "${valid}" 'agent B' 6 + ledger_earlier="${round1} ${round2} ${head} ${head} ${head}" + fct_readiness "${head}" Ready "${valid}" 'agent B' + ledger_earlier="${round1} ${round2}" fct_finish 1 fct_finish_said 'readiness.md: the ledger holds 6 rounds, and a build has at most 5' fct_finish_said 'this build used its 5 rounds. Stop, and report each open line and each open finding to the user' - fct_readiness "${head}" Ready "${valid}" 'agent B' 3 + fct_readiness "${head}" Ready "${valid}" 'agent B' stray="$(git -C "${finish}" commit-tree -m stray "${head}^{tree}")" - fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${first}..${head} | agent B | \`pending\` |" "| 3 | ${head} | agent A | ${stray}..${head} | agent B | \`pending\` |" + fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |" "| 3 | ${head} | agent A | ${stray}..${head} | agent B | \`pending\` |" fct_finish 1 fct_finish_said "the commit \"${stray}\" is not an ancestor of HEAD" - fct_readiness "${head}" Ready "${valid}" 'agent B' 3 - fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${first}..${head} | agent B | \`pending\` |" "| 3 | ${round2} | agent A | ${first}..${head} | agent B | \`pending\` |" + fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |" "| 3 | ${round2} | agent A | ${intake}..${head} | agent B | \`pending\` |" fct_finish 1 fct_finish_said "the last round names the head ${round2}, and the line Commit: names ${head}" # Round 1 commits its first scenario before its handler. - fct_readiness "${head}" Ready "${valid}" 'agent B' 3 - fct_edit "${finish}/readiness.md" "| Scenario | ${first} |" "| Scenario | ${head} |" + fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_edit "${finish}/readiness.md" "| Handler | ${round1} |" "| Handler | $(fct_step "${finish}" scenario) |" fct_finish 1 fct_finish_said 'the handler and the first scenario are in one commit' printf 'Commit: %s\nReviewer: agent B\nVerdict: Ready\n\n## Findings\n\n%s\n' "${head}" "${valid}" >"${finish}/readiness.md" @@ -86,17 +118,17 @@ fct_test_ledger() { fct_finish_said 'readiness.md: it holds no round under ## Rounds' # The word severity in prose, or a table of severities, fails with its cause (#416). - fct_readiness "${head}" Ready "${advise}"$'\n- **Evidence**: I kept the severity low'"${declined}"$'\n\n| Finding | Grade |\n|---|---|\n| R1-F1 | advise |' 'agent B' 3 + fct_readiness "${head}" Ready "${advise}"$'\n- **Evidence**: I kept the severity low'"${declined}"$'\n\n| Finding | Grade |\n|---|---|\n| R1-F1 | advise |' 'agent B' fct_finish 1 fct_finish_said 'this line names the word severity outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity' fct_finish_said 'this table names the severity advise' # A warn in prose beside an advise, or in a table of a part after ## Findings, fails (#416). - fct_readiness "${head}" Ready "${valid}"$'\n\nThis is a warn.\n\n## Notes\n\n| Long lines kept | `warn` |' 'agent B' 3 + fct_readiness "${head}" Ready "${valid}"$'\n\nThis is a warn.\n\n## Notes\n\n| Long lines kept | `warn` |' 'agent B' fct_finish 1 fct_finish_said 'this line names the severity warn outside the line - **Severity**: of a finding' fct_finish_said 'this table names the severity warn' # Finish reads the header in the first three lines, after a byte order mark (#416). - fct_readiness "${head}" Ready "${valid}" 'agent B' 3 + fct_readiness "${head}" Ready "${valid}" 'agent B' printf '# Readiness review\n' | cat - "${finish}/readiness.md" >"${scratch}/readiness.md" mv "${scratch}/readiness.md" "${finish}/readiness.md" fct_finish 1 @@ -106,14 +138,14 @@ fct_test_ledger() { mv "${scratch}/readiness.md" "${finish}/readiness.md" fct_finish 0 - # A new build, such as of another command, starts at its own intake. - # The findings of the build before it are no history of the new build. + # A new build, such as of another command, starts at its own intake, + # after a review with a passing verdict. The findings of the build before + # it are no history of the new build. git -C "${finish}" checkout -q readiness.md - printf 'Intake of the next build.\n' >>"${finish}/README.md" - git -C "${finish}" commit -qam 'intake of the next build' - first="$(git -C "${finish}" rev-parse HEAD)" + fct_steps "${finish}" printf 'Handler of the next build.\n' >>"${finish}/README.md" git -C "${finish}" commit -qam 'handler of the next build' - fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready "${advise}${declined}" 'agent B' 1 "${first}" + ledger_earlier="" + fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready "${advise}${declined}" 'agent B' fct_finish 0 } diff --git a/scripts/template-signoff-test.sh b/scripts/template-signoff-test.sh index a0dbbc15..b2c86d45 100644 --- a/scripts/template-signoff-test.sh +++ b/scripts/template-signoff-test.sh @@ -25,6 +25,7 @@ SCRIPT [[ ! -e "${scratch}/signed-sha" ]] || fct_fail 'just signoff called gh signoff before just finish passed' cp -R "${repo}" "${ready}" + fct_steps "${ready}" for file in vision.md spec.md qa.md; do grep -v '^Fill:' "${ready}/${file}" >"${scratch}/form.md" mv "${scratch}/form.md" "${ready}/${file}" diff --git a/sdk/extensionrepo/ledger.go b/sdk/extensionrepo/ledger.go index 1a70a70d..bca6aeec 100644 --- a/sdk/extensionrepo/ledger.go +++ b/sdk/extensionrepo/ledger.go @@ -35,19 +35,7 @@ type round struct { func ledgerProblems(readiness, reviewed string, findings []finding, spec string, limits []limit) []string { const fix = "Fix: give readiness.md a table under ## Rounds with the columns " + "Round, Head, Change review, Change commits, Readiness review, and Finish, one row for each round" - var problems []string - var rounds []round - order := map[string]round{} - for _, l := range SpecLines(readiness) { - switch { - case l.Part == "Rounds" && l.Kind == SpecHeader && !slices.Equal(l.Cells, roundColumns): - problems = append(problems, fmt.Sprintf("readiness.md:%d: the table of ## Rounds has other columns. %s", l.Number, fix)) - case l.Part == "Rounds" && l.Kind == SpecRow: - rounds = append(rounds, round{l.Number, l.Cells}) - case l.Part == "Order of round 1" && l.Kind == SpecRow && len(l.Cells) == 2: - order[l.Cells[0]] = round{l.Number, l.Cells} - } - } + rounds, order, problems := ledgerTables(readiness) if len(rounds) == 0 { return append(problems, "readiness.md: it holds no round under ## Rounds. "+fix) } @@ -66,13 +54,69 @@ func ledgerProblems(readiness, reviewed string, findings []finding, spec string, } problems = append(problems, orderProblems(order)...) problems = append(problems, stateProblems(findings, len(rounds), spec, limits)...) - // The build starts at its intake, so the reviews of an earlier build, - // such as of another command, are no history of this one. since := "" if intake, found := order["Intake"]; found { - since, _ = ledgerCommit(intake.cells[1]) + if commit, ok := ledgerCommit(intake.cells[1]); ok { + since = buildStart(commit) + } } - return append(problems, historyProblems(findings, since)...) + return append(problems, historyProblems(findings, rounds, since)...) +} + +// ledgerTables reads the tables of ## Rounds and ## Order of round 1. Each +// part ends at the next heading of level 1 or 2. Any other line in it fails, +// so no row hides behind a label or a heading. +func ledgerTables(readiness string) ([]round, map[string]round, []string) { + const fix = "Fix: give readiness.md a table under ## Rounds with the columns " + + "Round, Head, Change review, Change commits, Readiness review, and Finish, one row for each round" + var problems []string + var rounds []round + order := map[string]round{} + lines, part := strings.Split(readiness, "\n"), "" + for _, l := range SpecLines(readiness) { + if h := readinessHeading.FindStringSubmatch(lines[l.Number-1]); h != nil && len(h[1]) <= 2 { + part = h[2] + continue + } + if part != "Rounds" && part != "Order of round 1" { + continue + } + switch { + case l.Kind == SpecBlank || l.Kind == SpecSeparator: + case l.Kind == SpecHeader && part == "Rounds" && !slices.Equal(l.Cells, roundColumns): + problems = append(problems, fmt.Sprintf("readiness.md:%d: the table of ## Rounds has other columns. %s", l.Number, fix)) + case l.Kind == SpecHeader: + case l.Kind == SpecRow && part == "Rounds": + rounds = append(rounds, round{l.Number, l.Cells}) + case l.Kind == SpecRow && len(l.Cells) == 2: + order[l.Cells[0]] = round{l.Number, l.Cells} + default: + problems = append(problems, fmt.Sprintf("readiness.md:%d: this line of ## %s is no row of its table. Fix: keep only the table in this part", l.Number, part)) + } + } + return rounds, order, problems +} + +// buildStart returns the newest readiness.md committed at or before the +// intake with a passing verdict, which ended the build before this one. A +// review that needs work never ends a build, so moving the intake hides no +// history. It returns "" when no build ended before. +func buildStart(intake string) string { + log, err := exec.Command("git", "log", "--format=%H", intake, "--", "readiness.md").Output() + if err != nil { + return "" + } + for _, revision := range strings.Fields(string(log)) { + old, err := exec.Command("git", "show", revision+":readiness.md").Output() + if err != nil { + continue + } + header := strings.SplitN(strings.TrimPrefix(string(old), "\ufeff"), "\n", 4) + if v := readinessVerdict.FindStringSubmatch(strings.Join(header[:min(3, len(header))], "\n")); v != nil && (v[1] == "Ready" || v[1] == "Ready with stated limits") { + return revision + } + } + return "" } // roundProblems checks one row of ## Rounds, the round number of its place. @@ -132,6 +176,8 @@ func orderProblems(order map[string]round) []string { problems = append(problems, fmt.Sprintf("readiness.md:%d: the step %s comes before the step that precedes it in round 1. Fix: do the steps of round 1 in the order of the skill", r.line, step)) case step == "Handler" && commit == previous: problems = append(problems, fmt.Sprintf("readiness.md:%d: the handler and the first scenario are in one commit, so the scenario never failed first. Fix: commit the failing scenario before the handler", r.line)) + case commit == previous: + problems = append(problems, fmt.Sprintf("readiness.md:%d: the step %s is in the commit of the step before it. Fix: commit each of the first four steps of round 1 on its own", r.line, step)) } previous = commit } @@ -160,11 +206,6 @@ func ledgerRounds() int { if err != nil { return 0 } - n := 0 - for _, l := range SpecLines(string(data)) { - if l.Part == "Rounds" && l.Kind == SpecRow { - n++ - } - } - return n + rounds, _, _ := ledgerTables(string(data)) + return len(rounds) } diff --git a/sdk/extensionrepo/ledger_findings.go b/sdk/extensionrepo/ledger_findings.go index 7900d102..3ad7f12e 100644 --- a/sdk/extensionrepo/ledger_findings.go +++ b/sdk/extensionrepo/ledger_findings.go @@ -66,11 +66,11 @@ func stateProblems(findings []finding, n int, spec string, limits []limit) []str return problems } -// historyProblems compares the findings with each readiness.md committed -// after since, the intake of the build, or in all of HEAD without it. A -// finding keeps its ID in each later review, and its severity never goes -// down, so no loss leaves the ledger. -func historyProblems(findings []finding, since string) []string { +// historyProblems compares the ledger with each readiness.md committed after +// since, the end of the build before, or in all of HEAD without it. A finding +// keeps its ID in each later review, and its severity never goes down. Each +// round keeps its row, so no loss and no round leaves the ledger. +func historyProblems(findings []finding, rounds []round, since string) []string { revisions := "HEAD" if since != "" { revisions = since + "..HEAD" @@ -90,6 +90,7 @@ func historyProblems(findings []finding, since string) []string { if err != nil { continue } + problems = append(problems, roundHistory(rounds, strings.TrimPrefix(string(old), "\ufeff"), revision, reported)...) earlier, _ := readinessFindings(strings.TrimPrefix(string(old), "\ufeff")) for _, e := range earlier { now, kept := current[e.id] @@ -106,3 +107,24 @@ func historyProblems(findings []finding, since string) []string { } return problems } + +// roundHistory requires each round of an earlier committed ledger in the +// current one, with the same cells. Only a Finish that was pending changes. +func roundHistory(rounds []round, old, revision string, reported map[string]bool) []string { + var problems []string + earlier, _, _ := ledgerTables(old) + for i, e := range earlier { + key := "round " + strconv.Itoa(i+1) + if reported[key] || len(e.cells) != len(roundColumns) { + continue + } + kept := i < len(rounds) && len(rounds[i].cells) == len(roundColumns) && + slices.Equal(rounds[i].cells[:5], e.cells[:5]) && + (strings.Trim(e.cells[5], "`") == "pending" || rounds[i].cells[5] == e.cells[5]) + if !kept { + reported[key] = true + problems = append(problems, fmt.Sprintf("readiness.md: %s of the ledger in %.12s is missing or changed. Fix: keep the row of each earlier round as it was, and add one row for this round", key, revision)) + } + } + return problems +} diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index fc94932f..2147be78 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -85,6 +85,8 @@ After the findings, keep the ledger of the build. Add one row for this round, an Change commits names the commits that the change review covered. The readiness review covers Head. When one pass did both reviews, name the same reviewer in both cells. Finish holds the result of `just finish` for each earlier round, and `pending` for this round. +Keep the row of each earlier round as it was. Only its Finish changes, from `pending` to its result. A build has at most 5 rounds. +A new build, such as of another command, starts at a new intake, after a review with a passing verdict. Never reject a script because it is small, young, or private. Judge what the command does, not where the script comes from. diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index 91889fd1..5cbbbbbe 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -7,7 +7,7 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | Rule | Enforced by | |---|---| | Each running command executes in a scenario | [Kit executed calls][calls] | -| Call only through `ncly`, `ncly-{{ slug }}`, `exits`, `signal`, or `refuses` | [Kit call forms][links] | +| Call only through `ncly`, `ncly-{{ slug }}`, `exits`, `signal`, `refuses`, or `appears` | [Kit call forms][links] | | Each manifest example runs | [Kit examples][examples] | | Declare each own error and warning code, with its commands and modes | [`extension.Check`][declarations] | | Observe each declared code in its scope, or supply an `untested` reason with an issue | [Kit code coverage][codes] | diff --git a/template/spec.md.jinja b/template/spec.md.jinja index fc6a0400..cc358f18 100644 --- a/template/spec.md.jinja +++ b/template/spec.md.jinja @@ -33,7 +33,7 @@ The dry run makes the same free inspection and gives the same result. | ID | Condition | Code | |---|---|---| | `bad-input` | The file is missing or is no regular file, or `--timeout` is no duration | `USAGE_INVALID` | -| `no-python` | `python3` is missing or older than 3.9, or its version probe fails | `PREREQ_MISSING` | +| `no-python` | `python3` is missing or older than 3.9, or its version probe fails or does not finish within 10 s | `PREREQ_MISSING` | | `worker-fails` | `python3` exits with a code other than 0, its output is missing, malformed, or over 1 MB, or the timeout expires, even during the probe | `RUNTIME` | | `interrupted` | Ctrl-C arrives | `INTERRUPTED` | | `terminated` | SIGTERM arrives | `TERMINATED` | diff --git a/testdata/extensions/hello/testdata/script/wrap_stop.txtar b/testdata/extensions/hello/testdata/script/wrap_stop.txtar index 6797cf51..a660ad5b 100644 --- a/testdata/extensions/hello/testdata/script/wrap_stop.txtar +++ b/testdata/extensions/hello/testdata/script/wrap_stop.txtar @@ -83,6 +83,15 @@ answer stderr RUNTIME stderr '"message":"bash did not finish its work on input\.md within 1 ms, so it was stopped\.' ! exists worker.path +# A version probe that reaches its own limit of 10 s within a longer timeout is +# PREREQ_MISSING with exit 78, before work +env STUB=probe-hang +exits 78 ncly hello count input.md --timeout 15s --json +answer stderr PREREQ_MISSING +stderr '"message":"bash --version did not print its version in time\."' +gone probe.pid +! exists worker.path + # A version probe that fails is PREREQ_MISSING with exit 78, before work env STUB=probe-fail exits 78 ncly hello count input.md --timeout 1s --json From 7f3ae60e9c22386a4ef7ea5e7183aa54155fc86a Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 15:31:54 -0400 Subject: [PATCH 20/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20a=20for?= =?UTF-8?q?ged=20end=20of=20build,=20an=20uncommitted=20round,=20or=20a=20?= =?UTF-8?q?finding=20without=20an=20ID=20never=20leaves=20the=20ledger?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 2 found four ways past the ledger, and no way out for an honest build at the limit. A review ends a build only when its verdict follows from its findings and from the spec.md of its commit, so a changed verdict line ends nothing. Each committed review of the build names the head of a round, and each round before the last has a committed review of its head. An earlier finding without an ID keeps its title in later reviews. The user can now accept a limit at any round, also at the limit of 5: after a review, only the part **Limits.** of spec.md may change, and the reviewer gives the finding the state limit in a review of the same commit. Part of #417. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 8 +- scripts/template-finish-helpers.sh | 2 + scripts/template-ledger-builds-test.sh | 55 +++++++ scripts/template-ledger-test.sh | 53 +++--- scripts/template-test.sh | 2 + sdk/extensionrepo/ledger.go | 22 --- sdk/extensionrepo/ledger_findings.go | 65 -------- sdk/extensionrepo/ledger_history.go | 154 ++++++++++++++++++ sdk/extensionrepo/readiness.go | 8 +- sdk/extensionrepo/readiness_verdict.go | 79 ++++++--- .../.agents/skills/nicely-extension/SKILL.md | 1 + .../nicely-extension/readiness-review.md | 2 + 12 files changed, 304 insertions(+), 147 deletions(-) create mode 100644 scripts/template-ledger-builds-test.sh create mode 100644 sdk/extensionrepo/ledger_history.go diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index 8c7271a4..d627613a 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -10,7 +10,7 @@ This part says what `just finish` checks in an extension repository, as [Extensi - A line under `## Open lines` - A line under **Limits.** that is no table row with a spec ID, the state `limit`, and the user's acceptance - A `readiness.md` whose first three lines hold no `Commit: `, no `Reviewer: `, or no verdict `Ready` or `Ready with stated limits`. A byte order mark before them is ignored -- A reviewed commit that differs from the working tree outside `readiness.md` +- A reviewed commit that differs from the working tree outside `readiness.md`, and outside the part **Limits.** of `spec.md`. So the user accepts a limit after a review, with no new round - A finding of `readiness.md` without its own `###` heading under `## Findings`, or without one severity: `block`, `warn`, or `advise` - A severity after the first three lines, outside the line `- **Severity**:` of a finding, such as in a table, in fenced code, or in prose. Under `## Findings`, the word severity fails too. The message names the word or the table that caused it - `Ready` with a `block` or `warn` finding that is not `fixed`, or with a limit in `spec.md` @@ -55,10 +55,12 @@ A finding of the last round can have no state yet. - A finding without an ID, a finding of an earlier round without a state, or a state without its parts - A finding with the state `open`, even when `## Open lines` holds `None.` - A `warn` or `block` finding with the state `declined` -- A finding of an earlier committed `readiness.md` that is missing, or whose severity went down +- A finding of an earlier committed `readiness.md` that is missing, or whose severity went down. A finding without an ID keeps its title +- A committed review whose `Commit:` is the Head of no round, or a round before the last without a committed review of its Head - A round of an earlier committed ledger that is missing or changed. Only a Finish that was `pending` changes -The history of a build starts after the last committed `readiness.md` with a passing verdict at or before its Intake commit. +The history of a build starts after the last committed `readiness.md` that passes, at or before its Intake commit. +Such a review has a passing verdict that follows from its findings and from the `spec.md` of its commit. So an earlier build that ended is no history of this one. A review that needs work never ends a build, so a moved Intake hides no finding and resets no round. diff --git a/scripts/template-finish-helpers.sh b/scripts/template-finish-helpers.sh index c5088bf4..18c0910f 100644 --- a/scripts/template-finish-helpers.sh +++ b/scripts/template-finish-helpers.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash # Shared calls of the proof of just finish in template-finish-test.sh. # shellcheck disable=SC2154 # fct_test_finish sets finish and scratch for these helpers. +# Markdown backticks in the fixture text are literal. +# shellcheck disable=SC2016 # fct_finish runs just finish in the copy, and requires the exit code $1. fct_finish() { diff --git a/scripts/template-ledger-builds-test.sh b/scripts/template-ledger-builds-test.sh new file mode 100644 index 00000000..30fcf734 --- /dev/null +++ b/scripts/template-ledger-builds-test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Sourced by template-test.sh. fct_test_ledger calls fct_test_ledger_builds +# at the end of round 3, with its repository and its fixture variables. It +# proves the forms of readiness.md of #416, and the start of each build (#417). +# Markdown backticks in the fixture text are literal, and fct_readiness reads +# ledger_earlier. +# shellcheck disable=SC2016,SC2034,SC2154 + +fct_test_ledger_builds() { + # The word severity in prose, or a table of severities, fails with its cause (#416). + fct_readiness "${head}" Ready "${advise}"$'\n- **Evidence**: I kept the severity low'"${declined}"$'\n\n| Finding | Grade |\n|---|---|\n| R1-F1 | advise |' 'agent B' + fct_finish 1 + fct_finish_said 'this line names the word severity outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity' + fct_finish_said 'this table names the severity advise' + # A warn in prose beside an advise, or in a table of a part after ## Findings, fails (#416). + fct_readiness "${head}" Ready "${valid}"$'\n\nThis is a warn.\n\n## Notes\n\n| Long lines kept | `warn` |' 'agent B' + fct_finish 1 + fct_finish_said 'this line names the severity warn outside the line - **Severity**: of a finding' + fct_finish_said 'this table names the severity warn' + # Finish reads the header in the first three lines, after a byte order mark (#416). + fct_readiness "${head}" Ready "${valid}" 'agent B' + printf '# Readiness review\n' | cat - "${finish}/readiness.md" >"${scratch}/readiness.md" + mv "${scratch}/readiness.md" "${finish}/readiness.md" + fct_finish 1 + fct_finish_said 'readiness.md: its first three lines hold no line Commit: or Verdict: ' + git -C "${finish}" checkout -q readiness.md + printf '\357\273\277' | cat - "${finish}/readiness.md" >"${scratch}/readiness.md" + mv "${scratch}/readiness.md" "${finish}/readiness.md" + fct_finish 0 + + # A new build, such as of another command, starts at its own intake, + # after a review with a passing verdict. The findings of the build before + # it are no history of the new build. + git -C "${finish}" checkout -q readiness.md + fct_steps "${finish}" + printf 'Handler of the next build.\n' >>"${finish}/README.md" + git -C "${finish}" commit -qam 'handler of the next build' + ledger_earlier="" + fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready "${advise}${declined}" 'agent B' + fct_finish 0 + + # A review that needs work, committed again with the verdict Ready, ends no + # build. A finding without an ID keeps its title in later reviews. + fct_readiness "$(git -C "${finish}" rev-parse HEAD)" 'Needs work' "${warn/R2-F1 /}" + git -C "${finish}" add readiness.md + git -C "${finish}" commit -qm 'readiness review of the next build' + fct_edit "${finish}/readiness.md" 'Verdict: Needs work' 'Verdict: Ready' + git -C "${finish}" commit -qam 'the same review with the verdict Ready' + fct_steps "${finish}" + printf 'Handler of a third build.\n' >>"${finish}/README.md" + git -C "${finish}" commit -qam 'handler of a third build' + fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready 'None.' 'agent B' + fct_finish 1 + fct_finish_said 'readiness.md: the finding titled "Long lines are cut" of the review in' +} diff --git a/scripts/template-ledger-test.sh b/scripts/template-ledger-test.sh index 10d8801e..63c3b563 100644 --- a/scripts/template-ledger-test.sh +++ b/scripts/template-ledger-test.sh @@ -2,8 +2,9 @@ # Sourced by template-test.sh to prove the ledger of readiness.md, which # just finish reads (#417), and the forms of readiness.md of #416. It works on # a copy of the rendered repository, so the other tests keep their state. -# Markdown backticks in the fixture text are literal. -# shellcheck disable=SC2016 +# Markdown backticks in the fixture text are literal, and fct_readiness reads +# ledger_earlier. +# shellcheck disable=SC2016,SC2034 fct_test_ledger() { local finish="${2}/ledger" scratch="${2}" file intake round1 review1 round2 review2 head stray valid ledger_earlier="" @@ -65,6 +66,15 @@ fct_test_ledger() { git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'readiness review of round 3' fct_finish 0 + # The user accepts a limit with no new round: after the review, only the + # rows under **Limits.** of spec.md change. + fct_edit "${finish}/spec.md" '**Limits.** None.' $'**Limits.**\n\n| ID | State | Limit | Accepted |\n|---|---|---|---|\n| `long-lines` | `limit` | A line over 1 MB is cut | The user, 2026-10-31, https://example.com/pull/1 |' + fct_readiness "${head}" 'Ready with stated limits' "${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `limit`\n- **Spec**: `long-lines`' 'agent B' + fct_finish 0 + fct_edit "${finish}/spec.md" '| `bad-input` | The file is missing or is no regular file, or `--timeout` is no duration | `USAGE_INVALID` |' '| `bad-input` | The file is missing | `USAGE_INVALID` |' + fct_finish 1 + fct_finish_said 'and these files changed since: spec.md' + git -C "${finish}" checkout -q spec.md readiness.md # Moving the intake after a review that needs work hides no finding and resets no round. ledger_earlier="" @@ -82,6 +92,13 @@ fct_test_ledger() { ledger_earlier="${round1} ${round2}" fct_finish 1 fct_finish_said 'readiness.md: round 1 of the ledger in' + fct_finish_said "readiness.md: the review of ${round1:0:12}, committed in ${review1:0:12}, has no round under ## Rounds" + # Each round before the last has a committed review of its head. + ledger_earlier="${round1} ${round2} ${review2}" + fct_readiness "${head}" Ready "${valid}" 'agent B' + ledger_earlier="${round1} ${round2}" + fct_finish 1 + fct_finish_said 'round 3 has no committed review of its head' fct_readiness "${head}" Ready "${valid}" 'agent B' fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |" "| 3 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |"$'\n**Earlier build.**\n'"| 4 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |" fct_finish 1 @@ -117,35 +134,5 @@ fct_test_ledger() { fct_finish 1 fct_finish_said 'readiness.md: it holds no round under ## Rounds' - # The word severity in prose, or a table of severities, fails with its cause (#416). - fct_readiness "${head}" Ready "${advise}"$'\n- **Evidence**: I kept the severity low'"${declined}"$'\n\n| Finding | Grade |\n|---|---|\n| R1-F1 | advise |' 'agent B' - fct_finish 1 - fct_finish_said 'this line names the word severity outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity' - fct_finish_said 'this table names the severity advise' - # A warn in prose beside an advise, or in a table of a part after ## Findings, fails (#416). - fct_readiness "${head}" Ready "${valid}"$'\n\nThis is a warn.\n\n## Notes\n\n| Long lines kept | `warn` |' 'agent B' - fct_finish 1 - fct_finish_said 'this line names the severity warn outside the line - **Severity**: of a finding' - fct_finish_said 'this table names the severity warn' - # Finish reads the header in the first three lines, after a byte order mark (#416). - fct_readiness "${head}" Ready "${valid}" 'agent B' - printf '# Readiness review\n' | cat - "${finish}/readiness.md" >"${scratch}/readiness.md" - mv "${scratch}/readiness.md" "${finish}/readiness.md" - fct_finish 1 - fct_finish_said 'readiness.md: its first three lines hold no line Commit: or Verdict: ' - git -C "${finish}" checkout -q readiness.md - printf '\357\273\277' | cat - "${finish}/readiness.md" >"${scratch}/readiness.md" - mv "${scratch}/readiness.md" "${finish}/readiness.md" - fct_finish 0 - - # A new build, such as of another command, starts at its own intake, - # after a review with a passing verdict. The findings of the build before - # it are no history of the new build. - git -C "${finish}" checkout -q readiness.md - fct_steps "${finish}" - printf 'Handler of the next build.\n' >>"${finish}/README.md" - git -C "${finish}" commit -qam 'handler of the next build' - ledger_earlier="" - fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready "${advise}${declined}" 'agent B' - fct_finish 0 + fct_test_ledger_builds } diff --git a/scripts/template-test.sh b/scripts/template-test.sh index 76a899de..898148fb 100644 --- a/scripts/template-test.sh +++ b/scripts/template-test.sh @@ -138,6 +138,8 @@ fct_main() { source "${root}/scripts/template-finish-test.sh" # shellcheck source=scripts/template-ledger-test.sh source "${root}/scripts/template-ledger-test.sh" + # shellcheck source=scripts/template-ledger-builds-test.sh + source "${root}/scripts/template-ledger-builds-test.sh" # A Git hook or a caller can export GIT_DIR. The scratch repositories must see their own. while IFS= read -r variable; do unset "${variable}"; done < <(git rev-parse --local-env-vars) # just update runs the Copier that the template pins, and so does this test. diff --git a/sdk/extensionrepo/ledger.go b/sdk/extensionrepo/ledger.go index bca6aeec..4002487f 100644 --- a/sdk/extensionrepo/ledger.go +++ b/sdk/extensionrepo/ledger.go @@ -97,28 +97,6 @@ func ledgerTables(readiness string) ([]round, map[string]round, []string) { return rounds, order, problems } -// buildStart returns the newest readiness.md committed at or before the -// intake with a passing verdict, which ended the build before this one. A -// review that needs work never ends a build, so moving the intake hides no -// history. It returns "" when no build ended before. -func buildStart(intake string) string { - log, err := exec.Command("git", "log", "--format=%H", intake, "--", "readiness.md").Output() - if err != nil { - return "" - } - for _, revision := range strings.Fields(string(log)) { - old, err := exec.Command("git", "show", revision+":readiness.md").Output() - if err != nil { - continue - } - header := strings.SplitN(strings.TrimPrefix(string(old), "\ufeff"), "\n", 4) - if v := readinessVerdict.FindStringSubmatch(strings.Join(header[:min(3, len(header))], "\n")); v != nil && (v[1] == "Ready" || v[1] == "Ready with stated limits") { - return revision - } - } - return "" -} - // roundProblems checks one row of ## Rounds, the round number of its place. func roundProblems(r round, number int, last bool) []string { at := fmt.Sprintf("readiness.md:%d: ", r.line) diff --git a/sdk/extensionrepo/ledger_findings.go b/sdk/extensionrepo/ledger_findings.go index 3ad7f12e..8a5ca9a2 100644 --- a/sdk/extensionrepo/ledger_findings.go +++ b/sdk/extensionrepo/ledger_findings.go @@ -2,10 +2,8 @@ package extensionrepo import ( "fmt" - "os/exec" "slices" "strconv" - "strings" ) // stateProblems checks the state of each finding in the build of n rounds. @@ -65,66 +63,3 @@ func stateProblems(findings []finding, n int, spec string, limits []limit) []str } return problems } - -// historyProblems compares the ledger with each readiness.md committed after -// since, the end of the build before, or in all of HEAD without it. A finding -// keeps its ID in each later review, and its severity never goes down. Each -// round keeps its row, so no loss and no round leaves the ledger. -func historyProblems(findings []finding, rounds []round, since string) []string { - revisions := "HEAD" - if since != "" { - revisions = since + "..HEAD" - } - log, err := exec.Command("git", "log", "--format=%H", revisions, "--", "readiness.md").Output() - if err != nil { - return []string{"readiness.md: git log: " + err.Error()} - } - current := map[string]finding{} - for _, f := range findings { - current[f.id] = f - } - var problems []string - reported := map[string]bool{} - for _, revision := range strings.Fields(string(log)) { - old, err := exec.Command("git", "show", revision+":readiness.md").Output() - if err != nil { - continue - } - problems = append(problems, roundHistory(rounds, strings.TrimPrefix(string(old), "\ufeff"), revision, reported)...) - earlier, _ := readinessFindings(strings.TrimPrefix(string(old), "\ufeff")) - for _, e := range earlier { - now, kept := current[e.id] - switch { - case e.id == "" || reported[e.id]: - case !kept: - reported[e.id] = true - problems = append(problems, fmt.Sprintf("readiness.md: the finding %s of the review in %.12s is missing. Fix: keep each earlier finding under ## Findings, with its ID and its state", e.id, revision)) - case severityRank[now.severity] < severityRank[e.severity]: - reported[e.id] = true - problems = append(problems, fmt.Sprintf("readiness.md:%d: the finding %s had the severity %s in %.12s, and now has %s. Fix: keep its highest severity. A warn or block closes only as fixed or limit", now.line, e.id, e.severity, revision, now.severity)) - } - } - } - return problems -} - -// roundHistory requires each round of an earlier committed ledger in the -// current one, with the same cells. Only a Finish that was pending changes. -func roundHistory(rounds []round, old, revision string, reported map[string]bool) []string { - var problems []string - earlier, _, _ := ledgerTables(old) - for i, e := range earlier { - key := "round " + strconv.Itoa(i+1) - if reported[key] || len(e.cells) != len(roundColumns) { - continue - } - kept := i < len(rounds) && len(rounds[i].cells) == len(roundColumns) && - slices.Equal(rounds[i].cells[:5], e.cells[:5]) && - (strings.Trim(e.cells[5], "`") == "pending" || rounds[i].cells[5] == e.cells[5]) - if !kept { - reported[key] = true - problems = append(problems, fmt.Sprintf("readiness.md: %s of the ledger in %.12s is missing or changed. Fix: keep the row of each earlier round as it was, and add one row for this round", key, revision)) - } - } - return problems -} diff --git a/sdk/extensionrepo/ledger_history.go b/sdk/extensionrepo/ledger_history.go new file mode 100644 index 00000000..562b04bf --- /dev/null +++ b/sdk/extensionrepo/ledger_history.go @@ -0,0 +1,154 @@ +package extensionrepo + +import ( + "fmt" + "os/exec" + "slices" + "strconv" + "strings" +) + +// A review is one readiness.md committed in the history of the build. +type review struct { + revision, commit, text string +} + +// buildReviews returns each readiness.md committed after since, the end of +// the build before, or in all of HEAD without it, the newest first. +func buildReviews(since string) []review { + revisions := "HEAD" + if since != "" { + revisions = since + "..HEAD" + } + return committedReviews(revisions) +} + +func committedReviews(revisions string) []review { + log, err := exec.Command("git", "log", "--format=%H", revisions, "--", "readiness.md").Output() + if err != nil { + return nil + } + var reviews []review + for _, revision := range strings.Fields(string(log)) { + data, err := exec.Command("git", "show", revision+":readiness.md").Output() + if err != nil { + continue + } + r := review{revision: revision, text: strings.TrimPrefix(string(data), "\ufeff")} + if m := readinessCommit.FindStringSubmatch(readinessHeader(r.text)); m != nil { + r.commit, _ = ledgerCommit(m[1]) + } + reviews = append(reviews, r) + } + return reviews +} + +func readinessHeader(readiness string) string { + lines := strings.SplitN(readiness, "\n", 4) + return strings.Join(lines[:min(3, len(lines))], "\n") +} + +// buildStart returns the newest readiness.md committed at or before the +// intake that ended the build before this one, or "". Such a review passes: +// its verdict follows from its findings and from the spec.md of its commit. +// A review that needs work never ends a build, so moving the intake hides no +// history (#417). +func buildStart(intake string) string { + for _, r := range committedReviews(intake) { + v := readinessVerdict.FindStringSubmatch(readinessHeader(r.text)) + if v == nil || v[1] != "Ready" && v[1] != "Ready with stated limits" { + continue + } + spec, _ := exec.Command("git", "show", r.revision+":spec.md").Output() + open, limits := specProblems(string(spec)) + findings, unread := readinessFindings(r.text) + if len(open) == 0 && len(unread) == 0 && len(verdictProblems(v[1], findings, limits)) == 0 { + return r.revision + } + } + return "" +} + +// historyProblems compares the ledger with each review of the build. A +// finding keeps its ID, or its title when it had no ID, in each later review, +// and its severity never goes down. Each round keeps its row, and each +// round matches one committed review, so no loss and no round leaves it. +func historyProblems(findings []finding, rounds []round, since string) []string { + current := map[string]finding{} + for _, f := range findings { + current[f.id] = f + current["title "+f.title] = f + } + var problems []string + reported := map[string]bool{} + reviews := buildReviews(since) + for _, r := range reviews { + problems = append(problems, roundHistory(rounds, r.text, r.revision, reported)...) + earlier, _ := readinessFindings(r.text) + for _, e := range earlier { + key, name := e.id, e.id + if key == "" { + key, name = "title "+e.title, "titled "+strconv.Quote(e.title) + } + now, kept := current[key] + switch { + case reported[key]: + case !kept: + reported[key] = true + problems = append(problems, fmt.Sprintf("readiness.md: the finding %s of the review in %.12s is missing. Fix: keep each earlier finding under ## Findings, with its ID and its state", name, r.revision)) + case severityRank[now.severity] < severityRank[e.severity]: + reported[key] = true + problems = append(problems, fmt.Sprintf("readiness.md:%d: the finding %s had the severity %s in %.12s, and now has %s. Fix: keep its highest severity. A warn or block closes only as fixed or limit", now.line, name, e.severity, r.revision, now.severity)) + } + } + } + return append(problems, reviewedRounds(rounds, reviews)...) +} + +// reviewedRounds matches the rounds with the committed reviews of the build. +// Each review names the head of a round, and each round before the last has +// a committed review of its head. +func reviewedRounds(rounds []round, reviews []review) []string { + var heads, reviewed []string + for _, r := range rounds { + head := "" + if len(r.cells) == len(roundColumns) { + head, _ = ledgerCommit(r.cells[1]) + } + heads = append(heads, head) + } + var problems []string + for _, r := range reviews { + reviewed = append(reviewed, r.commit) + if r.commit != "" && !slices.Contains(heads, r.commit) && !slices.Contains(reviewed[:len(reviewed)-1], r.commit) { + problems = append(problems, fmt.Sprintf("readiness.md: the review of %.12s, committed in %.12s, has no round under ## Rounds. Fix: keep one row for each round, with the head that its review names", r.commit, r.revision)) + } + } + for i, head := range heads[:max(0, len(heads)-1)] { + if head != "" && !slices.Contains(reviewed, head) { + problems = append(problems, fmt.Sprintf("readiness.md:%d: round %d has no committed review of its head. Fix: commit readiness.md alone at the end of each round", rounds[i].line, i+1)) + } + } + return problems +} + +// roundHistory requires each round of an earlier committed ledger in the +// current one, with the same cells. Only a Finish that was pending changes. +func roundHistory(rounds []round, old, revision string, reported map[string]bool) []string { + var problems []string + earlier, _, _ := ledgerTables(old) + for i, e := range earlier { + key := "round " + strconv.Itoa(i+1) + if reported[key] || len(e.cells) != len(roundColumns) { + continue + } + kept := i < len(rounds) && len(rounds[i].cells) == len(roundColumns) && + slices.Equal(rounds[i].cells[:5], e.cells[:5]) && + (strings.Trim(e.cells[5], "`") == "pending" || rounds[i].cells[5] == e.cells[5]) + if !kept { + reported[key] = true + problems = append(problems, fmt.Sprintf("readiness.md: %s of the ledger in %.12s is missing or changed. Fix: keep the row of each earlier round as it was, and add one row for this round", key, revision)) + } + } + return problems +} diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index 556bffdf..7cd424b6 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -12,8 +12,8 @@ import ( // bullets - **State**:, - **Spec**:, - **Commit**:, and - **Reason**: give // its state in the ledger (#417). type finding struct { - line int - id, severity, state, spec, commit, why string + line int + id, title, severity, state, spec, commit, why string } var ( @@ -89,9 +89,9 @@ func readinessFindings(readiness string) ([]finding, []string) { part, seen = i+1, true } case level == 3 && part > 0: - f := finding{line: i + 1} + f := finding{line: i + 1, title: heading[2]} if id := readinessID.FindStringSubmatch(heading[2]); id != nil { - f.id = id[1] + f.id, f.title = id[1], strings.TrimSpace(heading[2][len(id[0]):]) } findings = append(findings, f) current = len(findings) - 1 diff --git a/sdk/extensionrepo/readiness_verdict.go b/sdk/extensionrepo/readiness_verdict.go index 8c1c6f35..101cec14 100644 --- a/sdk/extensionrepo/readiness_verdict.go +++ b/sdk/extensionrepo/readiness_verdict.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "os/exec" "slices" "strings" ) @@ -14,8 +15,7 @@ import ( // (#409, #417). Only readiness.md may differ from that commit, so the review // can be committed after it. Then the ledger of the build must hold. func readinessProblems(spec string, limits []limit) []string { - const fix = "Fix: ask an agent that did not write the code to run the readiness review of the nicely-extension skill, then start readiness.md with the lines Commit: , Reviewer: , and Verdict: " - const verdictFix = "Fix: write the verdict that the findings and the limits give, as the readiness review of the nicely-extension skill says" + const fix = readinessFix data, err := os.ReadFile("readiness.md") if errors.Is(err, os.ErrNotExist) { return []string{"readiness.md: no readiness verdict is recorded. " + fix} @@ -36,10 +36,45 @@ func readinessProblems(spec string, limits []limit) []string { if !readinessReviewer.MatchString(header) { problems = append(problems, "readiness.md: it names no reviewer. "+fix) } + problems = append(problems, verdictProblems(verdict[1], findings, limits)...) + sha, err := updateOutput("git", "rev-parse", "--verify", "--quiet", commit[1]+"^{commit}") + if err != nil { + return append(problems, fmt.Sprintf("readiness.md: Git knows no commit %s. %s", commit[1], fix)) + } + changed, err := updateOutput("git", "diff", "--name-only", commit[1], "--") + if err != nil { + return append(problems, "readiness.md: "+err.Error()) + } + added, err := updateOutput("git", "ls-files", "--others", "--exclude-standard") + if err != nil { + return append(problems, "readiness.md: "+err.Error()) + } + var files []string + for _, file := range strings.Split(string(changed)+string(added), "\n") { + if file != "" && file != "readiness.md" && !slices.Contains(files, file) && (file != "spec.md" || !limitsOnly(commit[1])) { + files = append(files, file) + } + } + if len(files) != 0 { + problems = append(problems, fmt.Sprintf("readiness.md: the review is of %s, and these files changed since: %s. Fix: run the readiness review again on this commit", commit[1], strings.Join(files, ", "))) + } + return append(problems, ledgerProblems(readiness, strings.TrimSpace(string(sha)), findings, spec, limits)...) +} + +const ( + readinessFix = "Fix: ask an agent that did not write the code to run the readiness review of the nicely-extension skill, then start readiness.md with the lines Commit: , Reviewer: , and Verdict: " + verdictFix = "Fix: write the verdict that the findings and the limits give, as the readiness review of the nicely-extension skill says" +) + +// verdictProblems reports a verdict that does not follow from the findings, +// their states, and the limits of spec.md (#409, #417). Finish also runs it +// on an earlier review that would end a build. +func verdictProblems(verdict string, findings []finding, limits []limit) []string { + var problems []string accepted := func(f finding) bool { return f.state == "limit" && slices.ContainsFunc(limits, func(l limit) bool { return l.id == f.spec && l.accepted }) } - switch v := verdict[1]; v { + switch v := verdict; v { case "Ready": for _, f := range findings { if (f.severity == "block" || f.severity == "warn") && f.state != "fixed" { @@ -62,28 +97,32 @@ func readinessProblems(spec string, limits []limit) []string { case "Needs work": problems = append(problems, "readiness.md: the verdict is Needs work. Fix: start the next round from its findings") default: - problems = append(problems, fmt.Sprintf("readiness.md: the verdict %q is not Ready, Ready with stated limits, or Needs work. %s", v, fix)) + problems = append(problems, fmt.Sprintf("readiness.md: the verdict %q is not Ready, Ready with stated limits, or Needs work. %s", v, readinessFix)) } - sha, err := updateOutput("git", "rev-parse", "--verify", "--quiet", commit[1]+"^{commit}") + return problems +} + +// limitsOnly reports whether spec.md differs from its version in commit only +// under **Limits.**. So the user accepts a limit after a review, and no new +// round starts for it (#417). +func limitsOnly(commit string) bool { + old, err := exec.Command("git", "show", commit+":spec.md").Output() if err != nil { - return append(problems, fmt.Sprintf("readiness.md: Git knows no commit %s. %s", commit[1], fix)) + return false } - changed, err := updateOutput("git", "diff", "--name-only", commit[1], "--") + current, err := os.ReadFile("spec.md") if err != nil { - return append(problems, "readiness.md: "+err.Error()) + return false } - added, err := updateOutput("git", "ls-files", "--others", "--exclude-standard") - if err != nil { - return append(problems, "readiness.md: "+err.Error()) - } - var files []string - for _, file := range strings.Split(string(changed)+string(added), "\n") { - if file != "" && file != "readiness.md" && !slices.Contains(files, file) { - files = append(files, file) + outside := func(spec string) []string { + lines := strings.Split(spec, "\n") + var kept []string + for _, l := range SpecLines(spec) { + if l.Part != "Limits" { + kept = append(kept, lines[l.Number-1]) + } } + return kept } - if len(files) != 0 { - problems = append(problems, fmt.Sprintf("readiness.md: the review is of %s, and these files changed since: %s. Fix: run the readiness review again on this commit", commit[1], strings.Join(files, ", "))) - } - return append(problems, ledgerProblems(readiness, strings.TrimSpace(string(sha)), findings, spec, limits)...) + return slices.Equal(outside(string(old)), outside(string(current))) } diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index 968c6c82..0837803e 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -113,6 +113,7 @@ At the limit of 5 rounds, stop. Do not start round 6. Report to the user each open line of `spec.md`, and each finding that the ledger records as `open`, with its ID. Then wait for the user. An answer that lets you choose, such as "decide yourself", accepts no limit and starts no round. +The user can accept each open finding as a limit. That needs no new round, as `readiness-review.md` says. ### Review, stop, and delivery diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 2147be78..ece97dae 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -57,6 +57,7 @@ End the review with one verdict. The verdict follows from the findings, and `jus Only the user accepts a limit. Ask the user, and wait for the answer. Then write the limit as a row under **Limits.** of its command, with the columns ID, State, Limit, and Accepted. +This change starts no round: the reviewer gives the finding the state `limit` in the review of the same commit. Its State is `limit`. Accepted records who accepted it, when, and where, such as a review comment. A loss that you report but that the user did not accept is a `warn` finding, not a limit. @@ -86,6 +87,7 @@ Change commits names the commits that the change review covered. The readiness r When one pass did both reviews, name the same reviewer in both cells. Finish holds the result of `just finish` for each earlier round, and `pending` for this round. Keep the row of each earlier round as it was. Only its Finish changes, from `pending` to its result. +Commit `readiness.md` at the end of each round, so each round has a committed review of its head. A build has at most 5 rounds. A new build, such as of another command, starts at a new intake, after a review with a passing verdict. From b3dc8b481f8a2fda9e554fed203fec83e73a9db7 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 15:32:06 -0400 Subject: [PATCH 21/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20template:=20the=20?= =?UTF-8?q?rule=20index=20names=20the=20checks=20of=20the=20ledger=20histo?= =?UTF-8?q?ry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- template/AGENTS/rule-index.md.jinja | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index 5cbbbbbe..97bb14bc 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -29,7 +29,9 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | At the limit, stop and report; an answer that lets the agent choose starts no round | Prose only | | A pull request for review, a stop report, and the delivery are three acts; deliver only after "Done when" | [Signoff][signoff] runs the [finish gate][finish], and [release][release] needs a receipt with `just finish` | | The ledger names the head, the reviewers, the commits that each review covered, and the result of `just finish` of each round | [Ledger][ledger] | -| Each finding keeps its ID and its highest severity, and gets a state each round; a `warn` or `block` closes only as `fixed` or `limit` | [Ledger findings][ledger-findings] | +| Each finding keeps its ID and its highest severity, and gets a state each round; a `warn` or `block` closes only as `fixed` or `limit` | [Ledger findings][ledger-findings], [ledger history][ledger-history] | +| Each round keeps its row and has a committed review; a build ends only at a review that passes | [Ledger history][ledger-history] | +| Accepting a limit starts no round; after a review, only **Limits.** of `spec.md` changes | [Finish gate][finish-verdict] | | A change of `spec.md` that narrows a promise to close a `warn` or `block` is a limit that only the user accepts | Prose only | | A command that replaces a user file proves its invariants with `protect`, `refuses`, `signal`, and `appears` | [Kit `refuses`][refuses], [kit `signal`][signal], [kit `appears`][appears] | @@ -46,6 +48,8 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". [finish]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/finish.go [ledger]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger.go [ledger-findings]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger_findings.go +[ledger-history]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger_history.go +[finish-verdict]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/readiness_verdict.go [signoff]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/signoff.go [release]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/release.go [template-test]: https://github.com/pascalandy/nicely/blob/main/scripts/template-test.sh From 38e1f05b990cfcfcbbffeb053f8abada356ced61 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 15:42:04 -0400 Subject: [PATCH 22/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20a=20bui?= =?UTF-8?q?ld=20ends=20only=20at=20a=20review=20that=20just=20finish=20wou?= =?UTF-8?q?ld=20pass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 3 found that a limit accepted after the review blocked every later build, that a review of the form of #409 never ended a build, and that a two-line edit of a review that needs work ended one. A review ends a build only when its verdict, its states, and the history of its own build pass, as just finish checks them. Its limits come from the spec.md of the next intake when only **Limits.** differs. A line - **Limit**: of the form of #409 reads as the state limit, and a review without a ledger needs a passing verdict only. The readiness review says that only readiness.md and **Limits.** of spec.md may differ from the reviewed commit, and that the limit is committed before the review. Part of #417. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 5 +- scripts/template-ledger-builds-test.sh | 56 ++++++++++++++---- sdk/extensionrepo/ledger_history.go | 58 ++++++++++++++----- sdk/extensionrepo/readiness.go | 6 +- sdk/extensionrepo/readiness_verdict.go | 11 ++-- .../nicely-extension/readiness-review.md | 4 +- 6 files changed, 107 insertions(+), 33 deletions(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index d627613a..db282fd7 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -60,7 +60,10 @@ A finding of the last round can have no state yet. - A round of an earlier committed ledger that is missing or changed. Only a Finish that was `pending` changes The history of a build starts after the last committed `readiness.md` that passes, at or before its Intake commit. -Such a review has a passing verdict that follows from its findings and from the `spec.md` of its commit. +Such a review passes as `just finish` would pass it: its verdict, its states, and the history of its own build. +Its limits come from the `spec.md` of the Intake when that file differs only under **Limits.**, so a limit accepted after the review counts. +A review without a ledger, from a template before the ledger, needs a passing verdict only. +Its form of #409, a line `- **Limit**:` with a spec ID, reads as the state `limit`. So an earlier build that ended is no history of this one. A review that needs work never ends a build, so a moved Intake hides no finding and resets no round. diff --git a/scripts/template-ledger-builds-test.sh b/scripts/template-ledger-builds-test.sh index 30fcf734..673fd9f0 100644 --- a/scripts/template-ledger-builds-test.sh +++ b/scripts/template-ledger-builds-test.sh @@ -7,6 +7,31 @@ # shellcheck disable=SC2016,SC2034,SC2154 fct_test_ledger_builds() { + local base forged limit=$'**Limits.**\n\n| ID | State | Limit | Accepted |\n|---|---|---|---|\n| `long-lines` | `limit` | A line over 1 MB is cut | The user, 2026-10-31, https://example.com/pull/1 |' + git -C "${finish}" checkout -q readiness.md spec.md + base="$(git -C "${finish}" rev-parse HEAD)" + # After a limit that the user accepted, or a review of the form of #409, + # a second build starts with a history of its own. + for forged in ledger legacy; do + if [[ "${forged}" == ledger ]]; then + fct_readiness "${head}" 'Ready with stated limits' "${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `limit`\n- **Spec**: `long-lines`' 'agent B' + else + printf 'Commit: %s\nReviewer: agent B\nVerdict: Ready with stated limits\n\n## Findings\n\n### Long lines are cut\n\n- **Severity**: `warn`\n- **Limit**: `long-lines`\n' "${head}" >"${finish}/readiness.md" + fi + git -C "${finish}" add readiness.md + git -C "${finish}" commit -qm "a review with an accepted limit: ${forged}" + fct_edit "${finish}/spec.md" '**Limits.** None.' "${limit}" + git -C "${finish}" commit -qam 'the user accepts the limit long-lines' + fct_steps "${finish}" + printf 'Handler of the build after the limit.\n' >>"${finish}/README.md" + git -C "${finish}" commit -qam 'handler of the build after the limit' + ledger_earlier="" + fct_readiness "$(git -C "${finish}" rev-parse HEAD)" 'Ready with stated limits' 'None.' 'agent B' + fct_finish 0 + ledger_earlier="${round1} ${round2}" + git -C "${finish}" reset -q --hard "${base}" + done + # The word severity in prose, or a table of severities, fails with its cause (#416). fct_readiness "${head}" Ready "${advise}"$'\n- **Evidence**: I kept the severity low'"${declined}"$'\n\n| Finding | Grade |\n|---|---|\n| R1-F1 | advise |' 'agent B' fct_finish 1 @@ -39,17 +64,28 @@ fct_test_ledger_builds() { fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready "${advise}${declined}" 'agent B' fct_finish 0 - # A review that needs work, committed again with the verdict Ready, ends no - # build. A finding without an ID keeps its title in later reviews. + # A review that needs work ends no build, even when its verdict line says + # Ready. A finding without an ID keeps its title in later reviews. fct_readiness "$(git -C "${finish}" rev-parse HEAD)" 'Needs work' "${warn/R2-F1 /}" git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'readiness review of the next build' - fct_edit "${finish}/readiness.md" 'Verdict: Needs work' 'Verdict: Ready' - git -C "${finish}" commit -qam 'the same review with the verdict Ready' - fct_steps "${finish}" - printf 'Handler of a third build.\n' >>"${finish}/README.md" - git -C "${finish}" commit -qam 'handler of a third build' - fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready 'None.' 'agent B' - fct_finish 1 - fct_finish_said 'readiness.md: the finding titled "Long lines are cut" of the review in' + base="$(git -C "${finish}" rev-parse HEAD)" + for forged in verdict state none; do + case "${forged}" in + verdict) fct_edit "${finish}/readiness.md" 'Verdict: Needs work' 'Verdict: Ready' ;; + state) + fct_edit "${finish}/readiness.md" 'Verdict: Needs work' 'Verdict: Ready' + fct_edit "${finish}/readiness.md" '- **Severity**: `warn`' $'- **Severity**: `warn`\n- **State**: `fixed`' + ;; + none) fct_readiness "$(git -C "${finish}" rev-parse HEAD~1)" Ready 'None.' 'agent B' ;; + esac + git -C "${finish}" commit -qam "a review that ends the build: ${forged}" + fct_steps "${finish}" + printf 'Handler of a third build.\n' >>"${finish}/README.md" + git -C "${finish}" commit -qam 'handler of a third build' + fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready 'None.' 'agent B' + fct_finish 1 + fct_finish_said 'readiness.md: the finding titled "Long lines are cut" of the review in' + git -C "${finish}" reset -q --hard "${base}" + done } diff --git a/sdk/extensionrepo/ledger_history.go b/sdk/extensionrepo/ledger_history.go index 562b04bf..5968eac2 100644 --- a/sdk/extensionrepo/ledger_history.go +++ b/sdk/extensionrepo/ledger_history.go @@ -13,16 +13,6 @@ type review struct { revision, commit, text string } -// buildReviews returns each readiness.md committed after since, the end of -// the build before, or in all of HEAD without it, the newest first. -func buildReviews(since string) []review { - revisions := "HEAD" - if since != "" { - revisions = since + "..HEAD" - } - return committedReviews(revisions) -} - func committedReviews(revisions string) []review { log, err := exec.Command("git", "log", "--format=%H", revisions, "--", "readiness.md").Output() if err != nil { @@ -49,20 +39,47 @@ func readinessHeader(readiness string) string { } // buildStart returns the newest readiness.md committed at or before the -// intake that ended the build before this one, or "". Such a review passes: -// its verdict follows from its findings and from the spec.md of its commit. -// A review that needs work never ends a build, so moving the intake hides no -// history (#417). +// intake that ended the build before this one, or "". A review that needs +// work never ends a build, so moving the intake hides no history (#417). func buildStart(intake string) string { + return endOfBuild(intake, 8) +} + +// endOfBuild finds the review that ended a build at or before intake. Such a +// review passes as just finish would pass it. Its verdict and its states +// follow from its findings. Its ledger keeps the history of its own build. +// Its limits come from the spec.md of the intake, when only **Limits.** +// differs. So a limit accepted after the review counts. A review without a +// ledger, of an older template, needs a passing verdict only. depth bounds +// the builds that it reads. +func endOfBuild(intake string, depth int) string { + if depth == 0 { + return "" + } + later, _ := exec.Command("git", "show", intake+":spec.md").Output() for _, r := range committedReviews(intake) { v := readinessVerdict.FindStringSubmatch(readinessHeader(r.text)) if v == nil || v[1] != "Ready" && v[1] != "Ready with stated limits" { continue } spec, _ := exec.Command("git", "show", r.revision+":spec.md").Output() + if sameOutsideLimits(string(spec), string(later)) { + spec = later + } open, limits := specProblems(string(spec)) findings, unread := readinessFindings(r.text) - if len(open) == 0 && len(unread) == 0 && len(verdictProblems(v[1], findings, limits)) == 0 { + problems := append(append(open, unread...), verdictProblems(v[1], findings, limits)...) + if rounds, order, _ := ledgerTables(r.text); len(rounds) != 0 { + problems = append(problems, stateProblems(findings, len(rounds), string(spec), limits)...) + since := "" + if step, found := order["Intake"]; found { + if commit, ok := ledgerCommit(step.cells[1]); ok { + since = endOfBuild(commit, depth-1) + } + } + problems = append(problems, historyAt(findings, rounds, since, r.revision+"^")...) + } + if len(problems) == 0 { return r.revision } } @@ -74,6 +91,11 @@ func buildStart(intake string) string { // and its severity never goes down. Each round keeps its row, and each // round matches one committed review, so no loss and no round leaves it. func historyProblems(findings []finding, rounds []round, since string) []string { + return historyAt(findings, rounds, since, "HEAD") +} + +// historyAt compares the ledger with the reviews of the build up to head. +func historyAt(findings []finding, rounds []round, since, head string) []string { current := map[string]finding{} for _, f := range findings { current[f.id] = f @@ -81,7 +103,11 @@ func historyProblems(findings []finding, rounds []round, since string) []string } var problems []string reported := map[string]bool{} - reviews := buildReviews(since) + revisions := head + if since != "" { + revisions = since + ".." + head + } + reviews := committedReviews(revisions) for _, r := range reviews { problems = append(problems, roundHistory(rounds, r.text, r.revision, reported)...) earlier, _ := readinessFindings(r.text) diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index 7cd424b6..115bed33 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -1,6 +1,7 @@ package extensionrepo import ( + "cmp" "fmt" "regexp" "slices" @@ -21,7 +22,7 @@ var ( readinessReviewer = regexp.MustCompile(`(?m)^Reviewer:[ \t]*\S`) readinessVerdict = regexp.MustCompile(`(?m)^Verdict: (.+?)\s*$`) readinessHeading = regexp.MustCompile(`^(#{1,6})\s+(.+?)\s*$`) - readinessPart = regexp.MustCompile(`^\s*[-*+]\s+\*\*(Severity|State|Spec|Commit|Reason)(?:\*\*:|:\*\*)\s*(.*?)\s*$`) + readinessPart = regexp.MustCompile(`^\s*[-*+]\s+\*\*(Severity|State|Spec|Commit|Reason|Limit)(?:\*\*:|:\*\*)\s*(.*?)\s*$`) readinessID = regexp.MustCompile(`^(R([1-9][0-9]*)-F[1-9][0-9]*)\b`) // readinessSeverity finds a severity in a form that finish cannot grade, // such as prose, code, or a table, anywhere after the header (#413, #416). @@ -119,6 +120,9 @@ func readinessFindings(readiness string) ([]finding, []string) { f.commit = value case "Reason": f.why = value + case "Limit": + // The form of #409 named an accepted limit in its own bullet. + f.state, f.spec = cmp.Or(f.state, "limit"), value } } case current < 0: diff --git a/sdk/extensionrepo/readiness_verdict.go b/sdk/extensionrepo/readiness_verdict.go index 101cec14..71fb7009 100644 --- a/sdk/extensionrepo/readiness_verdict.go +++ b/sdk/extensionrepo/readiness_verdict.go @@ -111,9 +111,12 @@ func limitsOnly(commit string) bool { return false } current, err := os.ReadFile("spec.md") - if err != nil { - return false - } + return err == nil && sameOutsideLimits(string(old), string(current)) +} + +// sameOutsideLimits reports whether two texts of spec.md differ only under +// **Limits.**. +func sameOutsideLimits(a, b string) bool { outside := func(spec string) []string { lines := strings.Split(spec, "\n") var kept []string @@ -124,5 +127,5 @@ func limitsOnly(commit string) bool { } return kept } - return slices.Equal(outside(string(old)), outside(string(current))) + return slices.Equal(outside(a), outside(b)) } diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index ece97dae..3a9ff9d9 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -14,7 +14,7 @@ Reviewer: Verdict: ``` -Then commit `readiness.md` alone. `just finish` accepts the verdict while only `readiness.md` differs from that commit. +Then commit `readiness.md` alone. `just finish` accepts the verdict while only `readiness.md` and the part **Limits.** of `spec.md` differ from that commit. `finish-gate.md` in the contract lists each check of `just finish`. ## Findings @@ -58,6 +58,7 @@ End the review with one verdict. The verdict follows from the findings, and `jus Only the user accepts a limit. Ask the user, and wait for the answer. Then write the limit as a row under **Limits.** of its command, with the columns ID, State, Limit, and Accepted. This change starts no round: the reviewer gives the finding the state `limit` in the review of the same commit. +Commit the limit in `spec.md`, then commit `readiness.md` alone. Its State is `limit`. Accepted records who accepted it, when, and where, such as a review comment. A loss that you report but that the user did not accept is a `warn` finding, not a limit. @@ -90,5 +91,6 @@ Keep the row of each earlier round as it was. Only its Finish changes, from `pen Commit `readiness.md` at the end of each round, so each round has a committed review of its head. A build has at most 5 rounds. A new build, such as of another command, starts at a new intake, after a review with a passing verdict. +After `just update` from a template without a ledger, commit a review in this form before the next intake. Never reject a script because it is small, young, or private. Judge what the command does, not where the script comes from. From 30546ad3afa37e28afd2522314f3a865636369bf Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 15:53:50 -0400 Subject: [PATCH 23/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20template?= =?UTF-8?q?:=20the=20ledger=20is=20cumulative,=20and=20a=20round=20with=20?= =?UTF-8?q?the=20Finish=20pass=20ends=20a=20build?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 4 found that the inferred end of a build failed at the 10th build, grew three times slower with each build, broke again when spec.md changed before the next intake, and let a 7-round build end. Three fixes of that inferred boundary failed in a row, so the ledger drops it. History is every committed readiness.md of HEAD. The round numbers go on from build to build, and the ledger keeps each finding. A round before the last whose Finish is pass ends a build, when its newest committed review passes: a verdict that follows from its findings, valid states with the current limits, and each finding of the reviews before it. Each build, also one that ended, has at most 5 rounds, and the intake of a build follows the end of the build before. A Limit line no longer replaces a Spec line. Part of #417. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 22 ++-- scripts/template-finish-helpers.sh | 18 ++- scripts/template-ledger-builds-test.sh | 63 ++++++--- scripts/template-ledger-test.sh | 23 +++- sdk/extensionrepo/ledger.go | 43 ++++-- sdk/extensionrepo/ledger_history.go | 124 ++++++++---------- sdk/extensionrepo/readiness.go | 2 +- .../nicely-extension/readiness-review.md | 5 +- 8 files changed, 172 insertions(+), 128 deletions(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index db282fd7..c8fece95 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -18,18 +18,21 @@ This part says what `just finish` checks in an extension repository, as [Extensi ## The ledger -`readiness.md` also holds the ledger of the build of a command, from the intake to "Done when" (#417). +`readiness.md` also holds the ledger of the repository (#417). +A build of a command goes from its intake to "Done when". +The ledger keeps each round and each finding of each build. The reviewer of each round updates it, and commits `readiness.md` alone. Under `## Rounds`, a table has the columns Round, Head, Change review, Change commits, Readiness review, and Finish. -Each row is one round, numbered from 1. +Each row is one round, numbered from 1. The numbers go on from one build to the next. Head is the full SHA of the reviewed commit. Change commits names the commits that the change review covered, as `..`. The readiness review covers Head. When one pass did both reviews, both cells name the same reviewer. Finish is `pass` or `fail` for an earlier round, and `pending` for the last round, whose result is this run. +A round before the last whose Finish is `pass` ends a build. The next round starts the next build. -Under `## Order of round 1`, a table with the columns Step and Commit names the commit of each step: Intake, Spec, Scenario, and Handler. +Under `## Order of round 1`, a table with the columns Step and Commit names the commit of each step of the current build: Intake, Spec, Scenario, and Handler. Each finding under `## Findings` starts its heading with a stable ID, such as `### R2-F3`. `R2` is the round that first reported it, and the ID stays in later rounds. @@ -48,7 +51,9 @@ A finding of the last round can have no state yet. - No round under `## Rounds`, or a row without its six cells, its number, a full Head, or both reviewers - A line under `## Rounds` or `## Order of round 1` that is no row of its table, such as a label that would hide rows -- More than 5 rounds. The message tells the agent to stop and to report to the user +- A build of more than 5 rounds, also a build that ended. The message tells the agent to stop and to report to the user +- A round with the Finish `pass` whose newest committed review does not pass. That review needs a passing verdict that follows from its findings, valid states, and each finding of the reviews before it +- An Intake that is not after the Head of the round that ended the build before - A commit of a round or of a fixed finding that is not HEAD or an ancestor of HEAD - A last round whose Head is not the `Commit:` of `readiness.md` - A step of round 1 that is missing, out of order, or in the commit of the step before it @@ -59,12 +64,9 @@ A finding of the last round can have no state yet. - A committed review whose `Commit:` is the Head of no round, or a round before the last without a committed review of its Head - A round of an earlier committed ledger that is missing or changed. Only a Finish that was `pending` changes -The history of a build starts after the last committed `readiness.md` that passes, at or before its Intake commit. -Such a review passes as `just finish` would pass it: its verdict, its states, and the history of its own build. -Its limits come from the `spec.md` of the Intake when that file differs only under **Limits.**, so a limit accepted after the review counts. -A review without a ledger, from a template before the ledger, needs a passing verdict only. +The history is every committed `readiness.md` of HEAD, so a moved Intake hides no finding and resets no round. +Finish checks a review that ends a build with the limits of the current `spec.md`. A finding with the state `limit` keeps its limit row, so the check holds after a limit that the user accepted later. +A review without a ledger, from a template before the ledger, needs a passing verdict only to end a build. Its form of #409, a line `- **Limit**:` with a spec ID, reads as the state `limit`. -So an earlier build that ended is no history of this one. -A review that needs work never ends a build, so a moved Intake hides no finding and resets no round. At 5 rounds, a failing `just finish` also tells the agent to stop and to report each open line and each open finding. diff --git a/scripts/template-finish-helpers.sh b/scripts/template-finish-helpers.sh index 18c0910f..045ba681 100644 --- a/scripts/template-finish-helpers.sh +++ b/scripts/template-finish-helpers.sh @@ -55,18 +55,22 @@ fct_step() { } # fct_ledger prints the ledger of the repository $1 for the rounds that -# reviewed the heads $2, in order. The newest commits of fct_steps give the -# order of round 1, and its handler is in the first head (#417). +# reviewed the heads $2, in order. A head written :pass gives its round +# the Finish pass, which ends its build. The newest commits of fct_steps give +# the order of round 1 of the last build, and its handler is in the first +# head of that build (#417). fct_ledger() { - local heads intake i finish_cell + local heads first i head finish_cell handler="" read -r -a heads <<<"${2}" - intake="$(fct_step "${1}" intake)" + first="$(git -C "${1}" rev-list --max-parents=0 HEAD)" printf '\n## Rounds\n\n| Round | Head | Change review | Change commits | Readiness review | Finish |\n|---|---|---|---|---|---|\n' for ((i = 0; i < ${#heads[@]}; i++)); do - finish_cell='`fail`' + head="${heads[i]%%:*}" finish_cell='`fail`' + [[ -n "${handler}" ]] || handler="${head}" + [[ "${heads[i]}" != *:pass ]] || finish_cell='`pass`' handler="" [[ "$((i + 1))" != "${#heads[@]}" ]] || finish_cell='`pending`' - printf '| %s | %s | agent A | %s..%s | agent B | %s |\n' "$((i + 1))" "${heads[i]}" "${intake}" "${heads[i]}" "${finish_cell}" + printf '| %s | %s | agent A | %s..%s | agent B | %s |\n' "$((i + 1))" "${head}" "${first}" "${head}" "${finish_cell}" done printf '\n## Order of round 1\n\n| Step | Commit |\n|---|---|\n' - printf '| %s | %s |\n' Intake "${intake}" Spec "$(fct_step "${1}" spec)" Scenario "$(fct_step "${1}" scenario)" Handler "${heads[0]:-}" + printf '| %s | %s |\n' Intake "$(fct_step "${1}" intake)" Spec "$(fct_step "${1}" spec)" Scenario "$(fct_step "${1}" scenario)" Handler "${handler}" } diff --git a/scripts/template-ledger-builds-test.sh b/scripts/template-ledger-builds-test.sh index 673fd9f0..477d2a81 100644 --- a/scripts/template-ledger-builds-test.sh +++ b/scripts/template-ledger-builds-test.sh @@ -7,15 +7,17 @@ # shellcheck disable=SC2016,SC2034,SC2154 fct_test_ledger_builds() { - local base forged limit=$'**Limits.**\n\n| ID | State | Limit | Accepted |\n|---|---|---|---|\n| `long-lines` | `limit` | A line over 1 MB is cut | The user, 2026-10-31, https://example.com/pull/1 |' + local base forged kept head2 head3 i limit=$'**Limits.**\n\n| ID | State | Limit | Accepted |\n|---|---|---|---|\n| `long-lines` | `limit` | A line over 1 MB is cut | The user, 2026-10-31, https://example.com/pull/1 |' git -C "${finish}" checkout -q readiness.md spec.md base="$(git -C "${finish}" rev-parse HEAD)" - # After a limit that the user accepted, or a review of the form of #409, - # a second build starts with a history of its own. + # After a limit that the user accepted, or a review of the form of #409, a + # round with the Finish pass ends the build, and the next build starts. for forged in ledger legacy; do if [[ "${forged}" == ledger ]]; then - fct_readiness "${head}" 'Ready with stated limits' "${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `limit`\n- **Spec**: `long-lines`' 'agent B' + kept="${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `limit`\n- **Spec**: `long-lines`' + fct_readiness "${head}" 'Ready with stated limits' "${kept}" 'agent B' else + kept="${valid}" printf 'Commit: %s\nReviewer: agent B\nVerdict: Ready with stated limits\n\n## Findings\n\n### Long lines are cut\n\n- **Severity**: `warn`\n- **Limit**: `long-lines`\n' "${head}" >"${finish}/readiness.md" fi git -C "${finish}" add readiness.md @@ -25,8 +27,8 @@ fct_test_ledger_builds() { fct_steps "${finish}" printf 'Handler of the build after the limit.\n' >>"${finish}/README.md" git -C "${finish}" commit -qam 'handler of the build after the limit' - ledger_earlier="" - fct_readiness "$(git -C "${finish}" rev-parse HEAD)" 'Ready with stated limits' 'None.' 'agent B' + ledger_earlier="${round1} ${round2} ${head}:pass" + fct_readiness "$(git -C "${finish}" rev-parse HEAD)" 'Ready with stated limits' "${kept}" 'agent B' fct_finish 0 ledger_earlier="${round1} ${round2}" git -C "${finish}" reset -q --hard "${base}" @@ -53,39 +55,58 @@ fct_test_ledger_builds() { mv "${scratch}/readiness.md" "${finish}/readiness.md" fct_finish 0 - # A new build, such as of another command, starts at its own intake, - # after a review with a passing verdict. The findings of the build before - # it are no history of the new build. + # A new build, such as of another command, starts at its own intake after + # a round with the Finish pass. The ledger keeps the findings of each build. git -C "${finish}" checkout -q readiness.md fct_steps "${finish}" printf 'Handler of the next build.\n' >>"${finish}/README.md" git -C "${finish}" commit -qam 'handler of the next build' - ledger_earlier="" - fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready "${advise}${declined}" 'agent B' + head2="$(git -C "${finish}" rev-parse HEAD)" ledger_earlier="${round1} ${round2} ${head}:pass" + fct_readiness "${head2}" Ready "${valid}" 'agent B' fct_finish 0 - - # A review that needs work ends no build, even when its verdict line says - # Ready. A finding without an ID keeps its title in later reviews. - fct_readiness "$(git -C "${finish}" rev-parse HEAD)" 'Needs work' "${warn/R2-F1 /}" git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'readiness review of the next build' base="$(git -C "${finish}" rev-parse HEAD)" + # Ten builds that each end with a review that passes still pass. + for i in 3 4 5 6 7 8 9 10; do + ledger_earlier="${ledger_earlier} $(git -C "${finish}" log --format=%H -n 1 HEAD~1):pass" + fct_steps "${finish}" + printf 'Handler of build %s.\n' "${i}" >>"${finish}/README.md" + git -C "${finish}" commit -qam "handler of build ${i}" + fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready "${valid}" 'agent B' + git -C "${finish}" add readiness.md + git -C "${finish}" commit -qm "readiness review of build ${i}" + done + fct_finish 0 + git -C "${finish}" reset -q --hard "${base}" + + # Only a review that passes ends a build: a verdict line that says Ready, + # a state with no commit, or a review that drops a finding ends none. + fct_steps "${finish}" + printf 'Handler of a build that needs work.\n' >>"${finish}/README.md" + git -C "${finish}" commit -qam 'handler of a build that needs work' + head3="$(git -C "${finish}" rev-parse HEAD)" ledger_earlier="${round1} ${round2} ${head}:pass ${head2}:pass" + fct_readiness "${head3}" 'Needs work' "${valid}"$'\n\n### Blank lines are dropped\n\n- **Severity**: `block`' 'agent B' + git -C "${finish}" add readiness.md + git -C "${finish}" commit -qm 'readiness review of the build that needs work' + base="$(git -C "${finish}" rev-parse HEAD)" for forged in verdict state none; do case "${forged}" in verdict) fct_edit "${finish}/readiness.md" 'Verdict: Needs work' 'Verdict: Ready' ;; state) fct_edit "${finish}/readiness.md" 'Verdict: Needs work' 'Verdict: Ready' - fct_edit "${finish}/readiness.md" '- **Severity**: `warn`' $'- **Severity**: `warn`\n- **State**: `fixed`' + fct_edit "${finish}/readiness.md" '- **Severity**: `block`' $'- **Severity**: `block`\n- **State**: `fixed`' ;; - none) fct_readiness "$(git -C "${finish}" rev-parse HEAD~1)" Ready 'None.' 'agent B' ;; + none) fct_readiness "${head3}" Ready 'None.' 'agent B' ;; esac git -C "${finish}" commit -qam "a review that ends the build: ${forged}" fct_steps "${finish}" - printf 'Handler of a third build.\n' >>"${finish}/README.md" - git -C "${finish}" commit -qam 'handler of a third build' - fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready 'None.' 'agent B' + printf 'Handler of the build after it.\n' >>"${finish}/README.md" + git -C "${finish}" commit -qam 'handler of the build after it' + ledger_earlier="${round1} ${round2} ${head}:pass ${head2}:pass ${head3}:pass" + fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready "${valid}"$'\n\n### R5-F1 Blank lines are dropped\n\n- **Severity**: `block`\n- **State**: `fixed`\n- **Commit**: '"${head3}"$'\n- **Spec**: `bad-input`' 'agent B' fct_finish 1 - fct_finish_said 'readiness.md: the finding titled "Long lines are cut" of the review in' + fct_finish_said 'round 5 says pass, but its review in' git -C "${finish}" reset -q --hard "${base}" done } diff --git a/scripts/template-ledger-test.sh b/scripts/template-ledger-test.sh index 63c3b563..a3ca70c2 100644 --- a/scripts/template-ledger-test.sh +++ b/scripts/template-ledger-test.sh @@ -7,14 +7,14 @@ # shellcheck disable=SC2016,SC2034 fct_test_ledger() { - local finish="${2}/ledger" scratch="${2}" file intake round1 review1 round2 review2 head stray valid ledger_earlier="" + local finish="${2}/ledger" scratch="${2}" file first intake round1 review1 round2 review2 head stray valid ledger_earlier="" local advise=$'### R1-F1 The count could name the longest line\n\n- **Severity**: `advise`' local declined=$'\n- **State**: `declined`\n- **Reason**: the answer stays small' local warn=$'### R2-F1 Long lines are cut\n\n- **Severity**: `warn`' local lowered=$'### R2-F1 Long lines are cut\n\n- **Severity**: `advise`' cp -R "${1}" "${finish}" fct_steps "${finish}" - intake="$(fct_step "${finish}" intake)" + intake="$(fct_step "${finish}" intake)" first="$(git -C "${finish}" rev-list --max-parents=0 HEAD)" for file in vision.md spec.md qa.md; do grep -v '^Fill:' "${finish}/${file}" >"${scratch}/form.md" mv "${scratch}/form.md" "${finish}/${file}" @@ -100,7 +100,7 @@ fct_test_ledger() { fct_finish 1 fct_finish_said 'round 3 has no committed review of its head' fct_readiness "${head}" Ready "${valid}" 'agent B' - fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |" "| 3 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |"$'\n**Earlier build.**\n'"| 4 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |" + fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${first}..${head} | agent B | \`pending\` |" "| 3 | ${head} | agent A | ${first}..${head} | agent B | \`pending\` |"$'\n**Earlier build.**\n'"| 4 | ${head} | agent A | ${first}..${head} | agent B | \`pending\` |" fct_finish 1 fct_finish_said 'this line of ## Rounds is no row of its table' # Each of the first four steps of round 1 has its own commit. @@ -114,15 +114,26 @@ fct_test_ledger() { fct_readiness "${head}" Ready "${valid}" 'agent B' ledger_earlier="${round1} ${round2}" fct_finish 1 - fct_finish_said 'readiness.md: the ledger holds 6 rounds, and a build has at most 5' + fct_finish_said 'readiness.md: build 1 of the ledger holds 6 rounds, and a build has at most 5' fct_finish_said 'this build used its 5 rounds. Stop, and report each open line and each open finding to the user' + # A build that ended keeps its limit, and only a review that passes ends a build. + ledger_earlier="${round1} ${round2} ${head} ${head} ${head} ${head}:pass" + fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said 'readiness.md: build 1 of the ledger holds 6 rounds, and a build has at most 5' + fct_finish_said 'the intake of this build comes before the end of the build before it' + ledger_earlier="${round1} ${round2}:pass" + fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said 'round 2 says pass, but its review in' + ledger_earlier="${round1} ${round2}" fct_readiness "${head}" Ready "${valid}" 'agent B' stray="$(git -C "${finish}" commit-tree -m stray "${head}^{tree}")" - fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |" "| 3 | ${head} | agent A | ${stray}..${head} | agent B | \`pending\` |" + fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${first}..${head} | agent B | \`pending\` |" "| 3 | ${head} | agent A | ${stray}..${head} | agent B | \`pending\` |" fct_finish 1 fct_finish_said "the commit \"${stray}\" is not an ancestor of HEAD" fct_readiness "${head}" Ready "${valid}" 'agent B' - fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${intake}..${head} | agent B | \`pending\` |" "| 3 | ${round2} | agent A | ${intake}..${head} | agent B | \`pending\` |" + fct_edit "${finish}/readiness.md" "| 3 | ${head} | agent A | ${first}..${head} | agent B | \`pending\` |" "| 3 | ${round2} | agent A | ${first}..${head} | agent B | \`pending\` |" fct_finish 1 fct_finish_said "the last round names the head ${round2}, and the line Commit: names ${head}" # Round 1 commits its first scenario before its handler. diff --git a/sdk/extensionrepo/ledger.go b/sdk/extensionrepo/ledger.go index 4002487f..4ed96df3 100644 --- a/sdk/extensionrepo/ledger.go +++ b/sdk/extensionrepo/ledger.go @@ -39,10 +39,6 @@ func ledgerProblems(readiness, reviewed string, findings []finding, spec string, if len(rounds) == 0 { return append(problems, "readiness.md: it holds no round under ## Rounds. "+fix) } - if len(rounds) > maxRounds { - problems = append(problems, fmt.Sprintf("readiness.md: the ledger holds %d rounds, and a build has at most %d. "+ - "Fix: stop, and report each open line and each open finding to the user. Do not start another round", len(rounds), maxRounds)) - } for i, r := range rounds { problems = append(problems, roundProblems(r, i+1, i == len(rounds)-1)...) } @@ -52,15 +48,33 @@ func ledgerProblems(readiness, reviewed string, findings []finding, spec string, "Fix: name the reviewed commit in both", last.line, last.cells[1], reviewed)) } } - problems = append(problems, orderProblems(order)...) + sizes, ended := builds(rounds) + for b, size := range sizes { + if size > maxRounds { + problems = append(problems, fmt.Sprintf("readiness.md: build %d of the ledger holds %d rounds, and a build has at most %d. "+ + "Fix: stop, and report each open line and each open finding to the user. Do not start another round", b+1, size, maxRounds)) + } + } + reviews := committedReviews() + problems = append(problems, orderProblems(order, ended)...) problems = append(problems, stateProblems(findings, len(rounds), spec, limits)...) - since := "" - if intake, found := order["Intake"]; found { - if commit, ok := ledgerCommit(intake.cells[1]); ok { - since = buildStart(commit) + problems = append(problems, passProblems(rounds, reviews, spec, limits)...) + return append(problems, historyProblems(findings, rounds, reviews)...) +} + +// builds splits the rounds into builds. A round before the last whose Finish +// is pass ends a build, and the next round starts the next one. It returns +// the number of rounds of each build, and the head of the last round of the +// build before the current one, or "". +func builds(rounds []round) ([]int, string) { + sizes, ended := []int{0}, "" + for i, r := range rounds { + sizes[len(sizes)-1]++ + if i < len(rounds)-1 && len(r.cells) == len(roundColumns) && strings.Trim(r.cells[5], "`") == "pass" { + sizes, ended = append(sizes, 0), r.cells[1] } } - return append(problems, historyProblems(findings, rounds, since)...) + return sizes, ended } // ledgerTables reads the tables of ## Rounds and ## Order of round 1. Each @@ -135,7 +149,7 @@ func roundProblems(r round, number int, last bool) []string { // orderProblems checks that round 1 recorded the intake, the spec, the // first scenario, and the handler, each in a commit after the one before. // The scenario precedes the handler, so it could fail first. -func orderProblems(order map[string]round) []string { +func orderProblems(order map[string]round, ended string) []string { const fix = "Fix: give readiness.md a table under ## Order of round 1 with the columns Step and Commit, and the rows Intake, Spec, Scenario, and Handler" var problems []string previous := "" @@ -150,6 +164,8 @@ func orderProblems(order map[string]round) []string { switch { case !ok: problems = append(problems, fmt.Sprintf("readiness.md:%d: the commit of the step %s is not an ancestor of HEAD. %s", r.line, step, fix)) + case step == "Intake" && ended != "" && ancestor(commit, strings.Trim(ended, "`")): + problems = append(problems, fmt.Sprintf("readiness.md:%d: the intake of this build comes before the end of the build before it, at %s. Fix: record the intake of this build in a new commit", r.line, ended)) case previous != "" && !ancestor(previous, commit): problems = append(problems, fmt.Sprintf("readiness.md:%d: the step %s comes before the step that precedes it in round 1. Fix: do the steps of round 1 in the order of the skill", r.line, step)) case step == "Handler" && commit == previous: @@ -178,12 +194,13 @@ func ancestor(commit, of string) bool { return exec.Command("git", "merge-base", "--is-ancestor", commit, of).Run() == nil } -// ledgerRounds counts the rounds of the ledger, for the note of finish. +// ledgerRounds counts the rounds of the current build, for the note of finish. func ledgerRounds() int { data, err := os.ReadFile("readiness.md") if err != nil { return 0 } rounds, _, _ := ledgerTables(string(data)) - return len(rounds) + sizes, _ := builds(rounds) + return sizes[len(sizes)-1] } diff --git a/sdk/extensionrepo/ledger_history.go b/sdk/extensionrepo/ledger_history.go index 5968eac2..8344a461 100644 --- a/sdk/extensionrepo/ledger_history.go +++ b/sdk/extensionrepo/ledger_history.go @@ -8,13 +8,16 @@ import ( "strings" ) -// A review is one readiness.md committed in the history of the build. +// A review is one readiness.md committed in the history of HEAD. type review struct { revision, commit, text string + rounds []round } -func committedReviews(revisions string) []review { - log, err := exec.Command("git", "log", "--format=%H", revisions, "--", "readiness.md").Output() +// committedReviews returns each readiness.md committed in HEAD, the newest +// first. The ledger is cumulative, so each one stays its history (#417). +func committedReviews() []review { + log, err := exec.Command("git", "log", "--format=%H", "HEAD", "--", "readiness.md").Output() if err != nil { return nil } @@ -28,6 +31,7 @@ func committedReviews(revisions string) []review { if m := readinessCommit.FindStringSubmatch(readinessHeader(r.text)); m != nil { r.commit, _ = ledgerCommit(m[1]) } + r.rounds, _, _ = ledgerTables(r.text) reviews = append(reviews, r) } return reviews @@ -38,78 +42,63 @@ func readinessHeader(readiness string) string { return strings.Join(lines[:min(3, len(lines))], "\n") } -// buildStart returns the newest readiness.md committed at or before the -// intake that ended the build before this one, or "". A review that needs -// work never ends a build, so moving the intake hides no history (#417). -func buildStart(intake string) string { - return endOfBuild(intake, 8) -} - -// endOfBuild finds the review that ended a build at or before intake. Such a -// review passes as just finish would pass it. Its verdict and its states -// follow from its findings. Its ledger keeps the history of its own build. -// Its limits come from the spec.md of the intake, when only **Limits.** -// differs. So a limit accepted after the review counts. A review without a -// ledger, of an older template, needs a passing verdict only. depth bounds -// the builds that it reads. -func endOfBuild(intake string, depth int) string { - if depth == 0 { - return "" - } - later, _ := exec.Command("git", "show", intake+":spec.md").Output() - for _, r := range committedReviews(intake) { - v := readinessVerdict.FindStringSubmatch(readinessHeader(r.text)) - if v == nil || v[1] != "Ready" && v[1] != "Ready with stated limits" { +// passProblems checks each round before the last whose Finish is pass. Such a +// round ends a build, so its newest committed review must pass. Its verdict +// and its states follow from its findings and from the limits of spec.md. It +// keeps each finding of the reviews before it. A review without a ledger, of +// a template before it, needs a passing verdict only. +func passProblems(rounds []round, reviews []review, spec string, limits []limit) []string { + var problems []string + for i, r := range rounds[:max(0, len(rounds)-1)] { + if len(r.cells) != len(roundColumns) || strings.Trim(r.cells[5], "`") != "pass" { + continue + } + head, _ := ledgerCommit(r.cells[1]) + at := slices.IndexFunc(reviews, func(v review) bool { return v.commit == head }) + if at < 0 { continue } - spec, _ := exec.Command("git", "show", r.revision+":spec.md").Output() - if sameOutsideLimits(string(spec), string(later)) { - spec = later + v := readinessVerdict.FindStringSubmatch(readinessHeader(reviews[at].text)) + findings, unread := readinessFindings(reviews[at].text) + failed := v == nil || len(unread) != 0 + if !failed { + failed = len(verdictProblems(v[1], findings, limits)) != 0 } - open, limits := specProblems(string(spec)) - findings, unread := readinessFindings(r.text) - problems := append(append(open, unread...), verdictProblems(v[1], findings, limits)...) - if rounds, order, _ := ledgerTables(r.text); len(rounds) != 0 { - problems = append(problems, stateProblems(findings, len(rounds), string(spec), limits)...) - since := "" - if step, found := order["Intake"]; found { - if commit, ok := ledgerCommit(step.cells[1]); ok { - since = endOfBuild(commit, depth-1) - } - } - problems = append(problems, historyAt(findings, rounds, since, r.revision+"^")...) + if !failed && len(reviews[at].rounds) != 0 { + failed = len(stateProblems(findings, len(reviews[at].rounds), spec, limits)) != 0 || + len(findingHistory(findings, reviews[at+1:], map[string]bool{})) != 0 } - if len(problems) == 0 { - return r.revision + if failed { + problems = append(problems, fmt.Sprintf("readiness.md:%d: round %d says pass, but its review in %.12s does not pass. Fix: write the result that just finish gave for that round", r.line, i+1, reviews[at].revision)) } } - return "" + return problems } -// historyProblems compares the ledger with each review of the build. A -// finding keeps its ID, or its title when it had no ID, in each later review, -// and its severity never goes down. Each round keeps its row, and each -// round matches one committed review, so no loss and no round leaves it. -func historyProblems(findings []finding, rounds []round, since string) []string { - return historyAt(findings, rounds, since, "HEAD") +// historyProblems compares the ledger with each committed review. Each round +// keeps its row, and each round matches one committed review, so no loss and +// no round leaves the ledger. +func historyProblems(findings []finding, rounds []round, reviews []review) []string { + var problems []string + reported := map[string]bool{} + for _, r := range reviews { + problems = append(problems, roundHistory(rounds, r, reported)...) + } + problems = append(problems, findingHistory(findings, reviews, reported)...) + return append(problems, reviewedRounds(rounds, reviews)...) } -// historyAt compares the ledger with the reviews of the build up to head. -func historyAt(findings []finding, rounds []round, since, head string) []string { +// findingHistory requires each finding of the reviews in findings. A finding +// keeps its ID, or its title when it had no ID, and its severity never goes +// down. +func findingHistory(findings []finding, reviews []review, reported map[string]bool) []string { current := map[string]finding{} for _, f := range findings { current[f.id] = f current["title "+f.title] = f } var problems []string - reported := map[string]bool{} - revisions := head - if since != "" { - revisions = since + ".." + head - } - reviews := committedReviews(revisions) for _, r := range reviews { - problems = append(problems, roundHistory(rounds, r.text, r.revision, reported)...) earlier, _ := readinessFindings(r.text) for _, e := range earlier { key, name := e.id, e.id @@ -128,12 +117,12 @@ func historyAt(findings []finding, rounds []round, since, head string) []string } } } - return append(problems, reviewedRounds(rounds, reviews)...) + return problems } -// reviewedRounds matches the rounds with the committed reviews of the build. -// Each review names the head of a round, and each round before the last has -// a committed review of its head. +// reviewedRounds matches the rounds with the committed reviews. Each review +// with a ledger names the head of a round, and each round before the last +// has a committed review of its head. func reviewedRounds(rounds []round, reviews []review) []string { var heads, reviewed []string for _, r := range rounds { @@ -145,10 +134,10 @@ func reviewedRounds(rounds []round, reviews []review) []string { } var problems []string for _, r := range reviews { - reviewed = append(reviewed, r.commit) - if r.commit != "" && !slices.Contains(heads, r.commit) && !slices.Contains(reviewed[:len(reviewed)-1], r.commit) { + if r.commit != "" && len(r.rounds) != 0 && !slices.Contains(heads, r.commit) && !slices.Contains(reviewed, r.commit) { problems = append(problems, fmt.Sprintf("readiness.md: the review of %.12s, committed in %.12s, has no round under ## Rounds. Fix: keep one row for each round, with the head that its review names", r.commit, r.revision)) } + reviewed = append(reviewed, r.commit) } for i, head := range heads[:max(0, len(heads)-1)] { if head != "" && !slices.Contains(reviewed, head) { @@ -160,10 +149,9 @@ func reviewedRounds(rounds []round, reviews []review) []string { // roundHistory requires each round of an earlier committed ledger in the // current one, with the same cells. Only a Finish that was pending changes. -func roundHistory(rounds []round, old, revision string, reported map[string]bool) []string { +func roundHistory(rounds []round, r review, reported map[string]bool) []string { var problems []string - earlier, _, _ := ledgerTables(old) - for i, e := range earlier { + for i, e := range r.rounds { key := "round " + strconv.Itoa(i+1) if reported[key] || len(e.cells) != len(roundColumns) { continue @@ -173,7 +161,7 @@ func roundHistory(rounds []round, old, revision string, reported map[string]bool (strings.Trim(e.cells[5], "`") == "pending" || rounds[i].cells[5] == e.cells[5]) if !kept { reported[key] = true - problems = append(problems, fmt.Sprintf("readiness.md: %s of the ledger in %.12s is missing or changed. Fix: keep the row of each earlier round as it was, and add one row for this round", key, revision)) + problems = append(problems, fmt.Sprintf("readiness.md: %s of the ledger in %.12s is missing or changed. Fix: keep the row of each earlier round as it was, and add one row for this round", key, r.revision)) } } return problems diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index 115bed33..cd3c5b74 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -122,7 +122,7 @@ func readinessFindings(readiness string) ([]finding, []string) { f.why = value case "Limit": // The form of #409 named an accepted limit in its own bullet. - f.state, f.spec = cmp.Or(f.state, "limit"), value + f.state, f.spec = cmp.Or(f.state, "limit"), cmp.Or(f.spec, value) } } case current < 0: diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 3a9ff9d9..28abc825 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -90,7 +90,8 @@ Finish holds the result of `just finish` for each earlier round, and `pending` f Keep the row of each earlier round as it was. Only its Finish changes, from `pending` to its result. Commit `readiness.md` at the end of each round, so each round has a committed review of its head. A build has at most 5 rounds. -A new build, such as of another command, starts at a new intake, after a review with a passing verdict. -After `just update` from a template without a ledger, commit a review in this form before the next intake. +A round whose Finish is `pass` ends a build. A new build, such as of another command, starts at a new intake after it. +The round numbers go on, and the ledger keeps each finding of each build. +After `just update` from a template without a ledger, keep each finding of the older reviews, with an ID and a state. Never reject a script because it is small, young, or private. Judge what the command does, not where the script comes from. From 5404c9f01a382f07a0b61f91c3e621ea896826b1 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 16:06:58 -0400 Subject: [PATCH 24/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20a=20rou?= =?UTF-8?q?nd=20with=20the=20Finish=20pass=20needs=20a=20review=20that=20j?= =?UTF-8?q?ust=20finish=20passed=20then?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round 5 found that finish checked a review that ended a build with the current spec.md, so a limit or a renamed spec ID of a later build broke an honest repository for good. A Finish pass on a Ready review whose finish failed, such as on an open line, also reset the 5 rounds. Finish now checks such a review as the repository stood when it was committed: no Fill: line, no open line, only readiness.md and **Limits.** of spec.md changed since its head, and a verdict and states that follow from its findings. A spec.md before the next round stands in when only **Limits.** differs. Each review is parsed once and each commit resolved once, so 80 builds take 0.76 s in place of 1.48 s. Part of #417. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 5 +- scripts/template-ledger-builds-test.sh | 37 ++++++++++- scripts/template-ledger-test.sh | 2 +- sdk/extensionrepo/ledger.go | 18 +++++- sdk/extensionrepo/ledger_history.go | 86 +++++++++++++++++++------ 5 files changed, 120 insertions(+), 28 deletions(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index c8fece95..f6f85f98 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -52,7 +52,7 @@ A finding of the last round can have no state yet. - No round under `## Rounds`, or a row without its six cells, its number, a full Head, or both reviewers - A line under `## Rounds` or `## Order of round 1` that is no row of its table, such as a label that would hide rows - A build of more than 5 rounds, also a build that ended. The message tells the agent to stop and to report to the user -- A round with the Finish `pass` whose newest committed review does not pass. That review needs a passing verdict that follows from its findings, valid states, and each finding of the reviews before it +- A round with the Finish `pass` whose newest committed review would fail `just finish` as the repository stood then. Its forms hold no `Fill:` line and no open line, and only `readiness.md` and **Limits.** of `spec.md` differ from its Head. Its verdict and states follow from its findings, which keep each finding of the reviews before it - An Intake that is not after the Head of the round that ended the build before - A commit of a round or of a fixed finding that is not HEAD or an ancestor of HEAD - A last round whose Head is not the `Commit:` of `readiness.md` @@ -65,7 +65,8 @@ A finding of the last round can have no state yet. - A round of an earlier committed ledger that is missing or changed. Only a Finish that was `pending` changes The history is every committed `readiness.md` of HEAD, so a moved Intake hides no finding and resets no round. -Finish checks a review that ends a build with the limits of the current `spec.md`. A finding with the state `limit` keeps its limit row, so the check holds after a limit that the user accepted later. +Finish checks a review that ends a build with the `spec.md` of its own commit. A later `spec.md` before the next round stands in when only **Limits.** differs, so a limit committed after the review counts. +A limit or a spec ID that a later build changes does not change that check. A review without a ledger, from a template before the ledger, needs a passing verdict only to end a build. Its form of #409, a line `- **Limit**:` with a spec ID, reads as the state `limit`. diff --git a/scripts/template-ledger-builds-test.sh b/scripts/template-ledger-builds-test.sh index 477d2a81..86e6fdf0 100644 --- a/scripts/template-ledger-builds-test.sh +++ b/scripts/template-ledger-builds-test.sh @@ -7,7 +7,7 @@ # shellcheck disable=SC2016,SC2034,SC2154 fct_test_ledger_builds() { - local base forged kept head2 head3 i limit=$'**Limits.**\n\n| ID | State | Limit | Accepted |\n|---|---|---|---|\n| `long-lines` | `limit` | A line over 1 MB is cut | The user, 2026-10-31, https://example.com/pull/1 |' + local base passed forged kept head2 head3 i limit=$'**Limits.**\n\n| ID | State | Limit | Accepted |\n|---|---|---|---|\n| `long-lines` | `limit` | A line over 1 MB is cut | The user, 2026-10-31, https://example.com/pull/1 |' git -C "${finish}" checkout -q readiness.md spec.md base="$(git -C "${finish}" rev-parse HEAD)" # After a limit that the user accepted, or a review of the form of #409, a @@ -66,7 +66,7 @@ fct_test_ledger_builds() { fct_finish 0 git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'readiness review of the next build' - base="$(git -C "${finish}" rev-parse HEAD)" + base="$(git -C "${finish}" rev-parse HEAD)" passed="${base}" # Ten builds that each end with a review that passes still pass. for i in 3 4 5 6 7 8 9 10; do ledger_earlier="${ledger_earlier} $(git -C "${finish}" log --format=%H -n 1 HEAD~1):pass" @@ -106,7 +106,38 @@ fct_test_ledger_builds() { ledger_earlier="${round1} ${round2} ${head}:pass ${head2}:pass ${head3}:pass" fct_readiness "$(git -C "${finish}" rev-parse HEAD)" Ready "${valid}"$'\n\n### R5-F1 Blank lines are dropped\n\n- **Severity**: `block`\n- **State**: `fixed`\n- **Commit**: '"${head3}"$'\n- **Spec**: `bad-input`' 'agent B' fct_finish 1 - fct_finish_said 'round 5 says pass, but its review in' + fct_finish_said 'round 5 says pass, but just finish fails on its review in' git -C "${finish}" reset -q --hard "${base}" done + + git -C "${finish}" reset -q --hard "${passed}" + # A review that ended a build passes as just finish read it then. A limit + # that a later build accepts changes nothing for it, and an open line of + # its time keeps it from ending the build. + for forged in limit open; do + if [[ "${forged}" == limit ]]; then + fct_edit "${finish}/spec.md" '**Limits.** None.' "${limit}" + git -C "${finish}" commit -qam 'a later build accepts the limit long-lines' + else + fct_edit "${finish}/spec.md" 'None.' $'| ID | Command | Part | Case | Code |\n|---|---|---|---|---|\n| `slow` | inspect | Failures | A file that the program reads slowly | `RUNTIME` |' + git -C "${finish}" commit -qam 'an open line' + head3="$(git -C "${finish}" rev-parse HEAD)" ledger_earlier="${round1} ${round2} ${head}:pass ${head2}:pass" + fct_readiness "${head3}" Ready "${valid}" 'agent B' + git -C "${finish}" add readiness.md + git -C "${finish}" commit -qm 'a review that misses the open line' + ledger_earlier="${ledger_earlier} ${head3}:pass" + fi + fct_steps "${finish}" + printf 'Handler of the build after it.\n' >>"${finish}/README.md" + git -C "${finish}" commit -qam 'handler of the build after it' + [[ "${forged}" == open ]] || ledger_earlier="${round1} ${round2} ${head}:pass ${head2}:pass" + fct_readiness "$(git -C "${finish}" rev-parse HEAD)" 'Ready with stated limits' "${valid}" 'agent B' + if [[ "${forged}" == limit ]]; then + fct_finish 0 + else + fct_finish 1 + fct_finish_said 'round 5 says pass, but just finish fails on its review in' + fi + git -C "${finish}" reset -q --hard "${passed}" + done } diff --git a/scripts/template-ledger-test.sh b/scripts/template-ledger-test.sh index a3ca70c2..1a92df69 100644 --- a/scripts/template-ledger-test.sh +++ b/scripts/template-ledger-test.sh @@ -125,7 +125,7 @@ fct_test_ledger() { ledger_earlier="${round1} ${round2}:pass" fct_readiness "${head}" Ready "${valid}" 'agent B' fct_finish 1 - fct_finish_said 'round 2 says pass, but its review in' + fct_finish_said 'round 2 says pass, but just finish fails on its review in' ledger_earlier="${round1} ${round2}" fct_readiness "${head}" Ready "${valid}" 'agent B' stray="$(git -C "${finish}" commit-tree -m stray "${head}^{tree}")" diff --git a/sdk/extensionrepo/ledger.go b/sdk/extensionrepo/ledger.go index 4ed96df3..7b81e531 100644 --- a/sdk/extensionrepo/ledger.go +++ b/sdk/extensionrepo/ledger.go @@ -58,7 +58,7 @@ func ledgerProblems(readiness, reviewed string, findings []finding, spec string, reviews := committedReviews() problems = append(problems, orderProblems(order, ended)...) problems = append(problems, stateProblems(findings, len(rounds), spec, limits)...) - problems = append(problems, passProblems(rounds, reviews, spec, limits)...) + problems = append(problems, passProblems(rounds, reviews)...) return append(problems, historyProblems(findings, rounds, reviews)...) } @@ -178,6 +178,15 @@ func orderProblems(order map[string]round, ended string) []string { return problems } +// A resolved commit of the ledger. One run of finish resolves each cell once, +// because the history repeats the same commits. +type resolved struct { + sha string + ok bool +} + +var resolvedCommits = map[string]resolved{} + // ledgerCommit resolves a commit of the ledger. It returns its full SHA, and // whether it is HEAD or an ancestor of HEAD. func ledgerCommit(cell string) (string, bool) { @@ -185,9 +194,14 @@ func ledgerCommit(cell string) (string, bool) { if m == nil { return "", false } + if r, found := resolvedCommits[m[1]]; found { + return r.sha, r.ok + } out, err := exec.Command("git", "rev-parse", "--verify", "--quiet", m[1]+"^{commit}").Output() sha := strings.TrimSpace(string(out)) - return sha, err == nil && ancestor(sha, "HEAD") + r := resolved{sha, err == nil && ancestor(sha, "HEAD")} + resolvedCommits[m[1]] = r + return r.sha, r.ok } func ancestor(commit, of string) bool { diff --git a/sdk/extensionrepo/ledger_history.go b/sdk/extensionrepo/ledger_history.go index 8344a461..ddd964b8 100644 --- a/sdk/extensionrepo/ledger_history.go +++ b/sdk/extensionrepo/ledger_history.go @@ -3,15 +3,21 @@ package extensionrepo import ( "fmt" "os/exec" + "regexp" "slices" "strconv" "strings" ) +// unfilled finds a line of a form that starts with Fill: after its indent. +var unfilled = regexp.MustCompile(`(?m)^[ \t]*Fill:`) + // A review is one readiness.md committed in the history of HEAD. type review struct { revision, commit, text string rounds []round + findings []finding + unread []string } // committedReviews returns each readiness.md committed in HEAD, the newest @@ -32,6 +38,7 @@ func committedReviews() []review { r.commit, _ = ledgerCommit(m[1]) } r.rounds, _, _ = ledgerTables(r.text) + r.findings, r.unread = readinessFindings(r.text) reviews = append(reviews, r) } return reviews @@ -43,11 +50,9 @@ func readinessHeader(readiness string) string { } // passProblems checks each round before the last whose Finish is pass. Such a -// round ends a build, so its newest committed review must pass. Its verdict -// and its states follow from its findings and from the limits of spec.md. It -// keeps each finding of the reviews before it. A review without a ledger, of -// a template before it, needs a passing verdict only. -func passProblems(rounds []round, reviews []review, spec string, limits []limit) []string { +// round ends a build, so just finish passed at its head. Its newest committed +// review must pass as just finish read it then. +func passProblems(rounds []round, reviews []review) []string { var problems []string for i, r := range rounds[:max(0, len(rounds)-1)] { if len(r.cells) != len(roundColumns) || strings.Trim(r.cells[5], "`") != "pass" { @@ -55,24 +60,66 @@ func passProblems(rounds []round, reviews []review, spec string, limits []limit) } head, _ := ledgerCommit(r.cells[1]) at := slices.IndexFunc(reviews, func(v review) bool { return v.commit == head }) - if at < 0 { - continue + next := "" + if len(rounds[i+1].cells) == len(roundColumns) { + next, _ = ledgerCommit(rounds[i+1].cells[1]) + } + if at >= 0 && !passedAt(reviews, at, head, next) { + problems = append(problems, fmt.Sprintf("readiness.md:%d: round %d says pass, but just finish fails on its review in %.12s. "+ + "Fix: write the result that just finish gave for that round", r.line, i+1, reviews[at].revision)) + } + } + return problems +} + +// passedAt reports whether just finish passed on the review reviews[at] of +// head, as the repository stood when the review was committed. The forms +// hold no Fill: line and no open line, and only readiness.md and **Limits.** +// of spec.md differ from head. A later spec.md before the next round can +// stand in when only **Limits.** differs, so a limit committed after the +// review counts. The verdict and the states follow from the findings, which +// keep each finding of the reviews before. A review without a ledger, of a +// template before it, needs no state and no history. +func passedAt(reviews []review, at int, head, next string) bool { + r := reviews[at] + show := func(commit, file string) (string, bool) { + data, err := exec.Command("git", "show", commit+":"+file).Output() + return string(data), err == nil && !unfilled.MatchString(string(data)) + } + for _, form := range []string{"vision.md", "qa.md"} { + if _, ok := show(r.revision, form); !ok { + return false } - v := readinessVerdict.FindStringSubmatch(readinessHeader(reviews[at].text)) - findings, unread := readinessFindings(reviews[at].text) - failed := v == nil || len(unread) != 0 - if !failed { - failed = len(verdictProblems(v[1], findings, limits)) != 0 + } + changed, err := exec.Command("git", "diff", "--name-only", head, r.revision, "--").Output() + if err != nil || slices.ContainsFunc(strings.Fields(string(changed)), func(f string) bool { return f != "readiness.md" && f != "spec.md" }) { + return false + } + v := readinessVerdict.FindStringSubmatch(readinessHeader(r.text)) + if v == nil || len(r.unread) != 0 { + return false + } + reviewed, _ := show(head, "spec.md") + candidates := []string{r.revision} + if next != "" { + later, _ := exec.Command("git", "log", "--format=%H", r.revision+".."+next, "--", "spec.md").Output() + candidates = append(candidates, strings.Fields(string(later))...) + } + for _, commit := range candidates { + spec, ok := show(commit, "spec.md") + if !ok || !sameOutsideLimits(reviewed, spec) { + continue } - if !failed && len(reviews[at].rounds) != 0 { - failed = len(stateProblems(findings, len(reviews[at].rounds), spec, limits)) != 0 || - len(findingHistory(findings, reviews[at+1:], map[string]bool{})) != 0 + open, limits := specProblems(spec) + if len(open) != 0 || len(verdictProblems(v[1], r.findings, limits)) != 0 { + continue } - if failed { - problems = append(problems, fmt.Sprintf("readiness.md:%d: round %d says pass, but its review in %.12s does not pass. Fix: write the result that just finish gave for that round", r.line, i+1, reviews[at].revision)) + if len(r.rounds) == 0 || len(stateProblems(r.findings, len(r.rounds), spec, limits)) == 0 && + len(findingHistory(r.findings, reviews[at+1:], map[string]bool{})) == 0 { + return true } } - return problems + return false } // historyProblems compares the ledger with each committed review. Each round @@ -99,8 +146,7 @@ func findingHistory(findings []finding, reviews []review, reported map[string]bo } var problems []string for _, r := range reviews { - earlier, _ := readinessFindings(r.text) - for _, e := range earlier { + for _, e := range r.findings { key, name := e.id, e.id if key == "" { key, name = "title "+e.title, "titled "+strconv.Quote(e.title) From 39bff544cbc3094c8a1c34ca351000b4c6587a73 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 17:10:40 -0400 Subject: [PATCH 25/61] =?UTF-8?q?=E2=9C=85=20test:=20sdk/extensiontest:=20?= =?UTF-8?q?the=20copy=20cases=20share=20one=20go=20test=20of=20the=20fixtu?= =?UTF-8?q?re?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each case of appears and protect ran its own go test of the standalone fixture, 29 in all. On the mini, that load made the 2 s budget of completion values and the 1 s timeout of a hello scenario fail in the full just check. The cases now run as scenarios of one go test per test function, and each case reads its own result and log. Co-Authored-By: Claude Opus 5.5 --- sdk/extensiontest/appears_test.go | 16 +----- sdk/extensiontest/protect_test.go | 89 +++++++++++++++++++------------ 2 files changed, 58 insertions(+), 47 deletions(-) diff --git a/sdk/extensiontest/appears_test.go b/sdk/extensiontest/appears_test.go index f37889b2..d073d529 100644 --- a/sdk/extensiontest/appears_test.go +++ b/sdk/extensiontest/appears_test.go @@ -31,7 +31,7 @@ rm -f "$tmp" func TestAppears(t *testing.T) { safe := appearsHost("", safeCommit) call := "appears started out/notes.txt ncly-standalone copy notes.md out/notes.txt\n" - cases := []struct{ name, host, script, problem string }{ + runCopyCases(t, []copyCase{ {"keeps the new file", safe, "protect notes.md out\n" + call + "stderr '\"code\":\"USAGE_INVALID\"'\n", ""}, {"through ncly", safe, "appears started out/notes.txt ncly standalone copy notes.md out/notes.txt\n", ""}, {"rename over the new file", appearsHost("", "mv -f \"$tmp\" \"$output\"\n"+copyRefusal), call, "the run replaced out/notes.txt"}, @@ -48,17 +48,5 @@ func TestAppears(t *testing.T) { {"force", safe, "appears started out/notes.txt ncly-standalone copy notes.md out/notes.txt --force\n", "remove --force"}, {"answer schema", appearsHost("", "rm -f \"$tmp\"\n"+strings.Replace(copyRefusal, `"ok":false`, `"ok":"false"`, 1)), call, "breaks the schema"}, {"path bypass", safe, "appears started out/notes.txt $XDG_DATA_HOME/nicely/extensions/ncly-standalone copy notes.md out/notes.txt\n", "use the registered command"}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - out, err := copyFixture(t, c.host, c.script) - if c.problem == "" { - if err != nil { - t.Fatalf("valid appears scenario failed: %v\n%s", err, out) - } - } else if err == nil || !strings.Contains(out, c.problem) { - t.Fatalf("want %q, got %v\n%s", c.problem, err, out) - } - }) - } + }) } diff --git a/sdk/extensiontest/protect_test.go b/sdk/extensiontest/protect_test.go index ff1e007d..4510962c 100644 --- a/sdk/extensiontest/protect_test.go +++ b/sdk/extensiontest/protect_test.go @@ -1,7 +1,9 @@ package extensiontest_test import ( + "fmt" "os" + "regexp" "strings" "testing" ) @@ -40,14 +42,59 @@ const copyRefusal = `echo '{"ok":false,"contract_version":1,"retry_safe":true,"e exit 2 ` -func copyFixture(t *testing.T, host, script string) (string, error) { +// A copyCase is one scenario of standalone copy, with its shell host. Its +// problem is a text of its failure, or "" for a scenario that passes. +type copyCase struct{ name, host, script, problem string } + +// runCopyCases runs each case as its own scenario in one copy of the +// standalone fixture, so the cases share one build and one go test. +func runCopyCases(t *testing.T, cases []copyCase) { t.Helper() ext := "$XDG_DATA_HOME/nicely/extensions/ncly-standalone" setup := "mkdir out\nrm " + ext + "\ncp host.sh " + ext + "\nchmod 755 " + ext + "\n" - return goTest(t, "standalone", os.Getenv("PATH"), - edit{"manifest.toml", "[programs.tally-tool]", copyCommand}, - edit{"extension.go", `"standalone fetch": fetch,`, "\"standalone fetch\": fetch,\n\t\t\"standalone copy\": tally,"}, - edit{file: "testdata/script/copy.txtar", new: setup + script + "\n-- notes.md --\nFirst line\nLast line\n-- host.sh --\n" + host}) + edits := []edit{ + {"manifest.toml", "[programs.tally-tool]", copyCommand}, + {"extension.go", `"standalone fetch": fetch,`, "\"standalone fetch\": fetch,\n\t\t\"standalone copy\": tally,"}, + } + for i, c := range cases { + edits = append(edits, edit{file: fmt.Sprintf("testdata/script/copy%02d.txtar", i), new: setup + c.script + "\n-- notes.md --\nFirst line\nLast line\n-- host.sh --\n" + c.host}) + } + out, _ := goTest(t, "standalone", os.Getenv("PATH"), edits...) + for i, c := range cases { + t.Run(c.name, func(t *testing.T) { + status, log := scenarioResult(out, fmt.Sprintf("copy%02d", i)) + switch { + case status == "": + t.Fatalf("no result for the scenario of %q\n%s", c.name, out) + case c.problem == "" && status != "PASS": + t.Fatalf("valid scenario failed\n%s", log) + case c.problem != "" && (status != "FAIL" || !strings.Contains(log, c.problem)): + t.Fatalf("want %q, got %s\n%s", c.problem, status, log) + } + }) + } +} + +// scenarioResult returns PASS or FAIL for the scenario name in the output of +// go test -v, and its log. Go prints that log under the lines === RUN, +// === CONT, or === NAME of the scenario, and its result at the end. +func scenarioResult(out, name string) (string, string) { + result := regexp.MustCompile(`(?m)^\s*--- (PASS|FAIL): \S*/` + name + ` \(`).FindStringSubmatch(out) + if result == nil { + return "", "" + } + var log strings.Builder + mine := false + for _, line := range strings.Split(out, "\n") { + if header := strings.TrimSpace(line); strings.HasPrefix(header, "=== ") || strings.HasPrefix(header, "--- ") { + mine = strings.HasPrefix(header, "=== ") && strings.HasSuffix(header, "/"+name) + continue + } + if mine { + log.WriteString(line + "\n") + } + } + return result[1], log.String() } // signalCopyHost copies its input to a temporary file, writes the marker, and waits. @@ -70,7 +117,7 @@ func TestProtectSignal(t *testing.T) { call := "signal TERM started ncly-standalone copy notes.md out/notes.txt\n" again := "rm started\n" + call old := "cp notes.md out/notes.txt\n" - cases := []struct{ name, host, script, problem string }{ + runCopyCases(t, []copyCase{ {"clean interruption", clean, "protect notes.md out\n" + call, ""}, {"marker in a protected folder", clean, "protect notes.md .\n" + call, ""}, {"temporary file left", signalCopyHost(":"), "protect notes.md out\n" + call, "signal: TERM changed protected paths"}, @@ -79,19 +126,7 @@ func TestProtectSignal(t *testing.T) { {"input changed", signalCopyHost(`rm -f "$tmp"; echo more >> "$input"`), "protect notes.md\n" + call, "signal: TERM changed protected paths"}, {"selection ends after one signal", signalCopyHost(`rm -f "$tmp"; [ -z "$DIRTY" ] || echo partial > "$output"`), "protect notes.md out\n" + call + "env DIRTY=1\n" + again, ""}, {"new selection for each signal", signalCopyHost(`rm -f "$tmp"; [ -z "$DIRTY" ] || echo partial > "$output"`), "protect notes.md out\n" + call + "env DIRTY=1\nprotect notes.md out\n" + again, "signal: TERM changed protected paths"}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - out, err := copyFixture(t, c.host, c.script) - if c.problem == "" { - if err != nil { - t.Fatalf("valid signal scenario failed: %v\n%s", err, out) - } - } else if err == nil || !strings.Contains(out, c.problem) { - t.Fatalf("want %q, got %v\n%s", c.problem, err, out) - } - }) - } + }) } // linkHost refuses an output that is a link. Before the refusal, the run part runs write. @@ -110,7 +145,7 @@ input=$1 output=$2 func TestProtectLink(t *testing.T) { ext := "$XDG_DATA_HOME/nicely/extensions/ncly-standalone" refusal := "symlink out/notes.txt -> ../notes.md\nprotect notes.md out/notes.txt\nrefuses USAGE_INVALID ncly-standalone copy notes.md out/notes.txt\n" - cases := []struct{ name, host, script, problem string }{ + runCopyCases(t, []copyCase{ {"refusal keeps the link", linkHost(":"), refusal, ""}, {"write through the link", linkHost(`echo copy >> "$output"`), refusal, "run changed protected paths"}, {"truncate through the link", linkHost(`: > "$output"`), refusal, "run changed protected paths"}, @@ -118,17 +153,5 @@ func TestProtectLink(t *testing.T) { {"selection stays after a signal", linkHost(`echo copy >> "$output"`), "symlink out/notes.txt -> ../notes.md\nprotect notes.md out/notes.txt\ncp signal.sh " + ext + "\n" + "signal TERM started ncly-standalone copy notes.md out/other.txt\ncp host.sh " + ext + "\nrefuses USAGE_INVALID ncly-standalone copy notes.md out/notes.txt\n" + "-- signal.sh --\n" + signalCopyHost(`rm -f "$tmp"`), "run changed protected paths"}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - out, err := copyFixture(t, c.host, c.script) - if c.problem == "" { - if err != nil { - t.Fatalf("valid link scenario failed: %v\n%s", err, out) - } - } else if err == nil || !strings.Contains(out, c.problem) { - t.Fatalf("want %q, got %v\n%s", c.problem, err, out) - } - }) - } + }) } From d4dd0030f5e16666797aa2f6b080694eb496e3b1 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 19:49:42 -0400 Subject: [PATCH 26/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20scripts:?= =?UTF-8?q?=20move=20the=20proof=20of=20the=20release=20URL=20to=20its=20o?= =?UTF-8?q?wn=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit template-release-test.sh is near the token limit of a file, and the next commit adds the proof of a forged receipt to it. The proof of the repository URL of a module with a major version suffix is its own responsibility. It moves, unchanged, to template-release-url-test.sh. Co-Authored-By: Claude Opus 5.5 --- scripts/template-release-test.sh | 14 ++------------ scripts/template-release-url-test.sh | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 scripts/template-release-url-test.sh diff --git a/scripts/template-release-test.sh b/scripts/template-release-test.sh index 7a514dbc..0ddaad6a 100644 --- a/scripts/template-release-test.sh +++ b/scripts/template-release-test.sh @@ -3,6 +3,8 @@ # shellcheck source=scripts/template-release-helpers.sh source "${BASH_SOURCE[0]%/*}/template-release-helpers.sh" +# shellcheck source=scripts/template-release-url-test.sh +source "${BASH_SOURCE[0]%/*}/template-release-url-test.sh" fct_test_release() { local repo="${1}" scratch="${2}" slug="${3}" extension_root="${4}" sha previous receipt dir status out_file out_bytes err_file err_bytes i examples command @@ -116,15 +118,3 @@ SCRIPT [[ -z "$(git -C "${repo}" status --porcelain)" ]] || fct_fail 'release wrote into tracked files' printf 'GREEN release receipt: correct README, Economy, and NOT_FOUND, exit 0\n' } - -# A Go module's /v2 suffix is not part of its published repository URL. -fct_test_release_url() { - local template_src="${1}" scratch="${2}" copier - shift 2 - copier=("$@") - "${copier[@]}" copy --quiet --vcs-ref v0.0.1 --defaults --data domain=probe --data module=example.com/ncly-probe/v2 "${template_src}" "${scratch}/ncly-probe-v2" - fct_has "${scratch}/ncly-probe-v2/README.md" 'git clone --branch TAG https://example.com/ncly-probe ncly-probe' - if grep -qF 'example.com/ncly-probe/v2' "${scratch}/ncly-probe-v2/README.md"; then - fct_fail 'README.md gives the module path example.com/ncly-probe/v2 as a repository URL' - fi -} diff --git a/scripts/template-release-url-test.sh b/scripts/template-release-url-test.sh new file mode 100644 index 00000000..85071fe6 --- /dev/null +++ b/scripts/template-release-url-test.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Sourced by template-release-test.sh. It proves the repository URL that a rendered +# README.md gives for a module path with a major version suffix. + +# A Go module's /v2 suffix is not part of its published repository URL. +fct_test_release_url() { + local template_src="${1}" scratch="${2}" copier + shift 2 + copier=("$@") + "${copier[@]}" copy --quiet --vcs-ref v0.0.1 --defaults --data domain=probe --data module=example.com/ncly-probe/v2 "${template_src}" "${scratch}/ncly-probe-v2" + fct_has "${scratch}/ncly-probe-v2/README.md" 'git clone --branch TAG https://example.com/ncly-probe ncly-probe' + if grep -qF 'example.com/ncly-probe/v2' "${scratch}/ncly-probe-v2/README.md"; then + fct_fail 'README.md gives the module path example.com/ncly-probe/v2 as a repository URL' + fi +} From 7ef57ef0331cddbf153d6f013c3276e32473a783 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 19:50:23 -0400 Subject: [PATCH 27/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20finish?= =?UTF-8?q?=20reads=20plain=20words=20as=20words,=20and=20release=20runs?= =?UTF-8?q?=20finish=20again?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finish read the words block, warn, and advise anywhere after the header as a severity. A title such as "Large inputs block the terminal", or prose such as "the CLI does not warn", failed it. Finish now fails only on a severity in code, such as `warn`, on a severity alone in a cell of a table, and on the word severity under ## Findings. The plain words pass. Release trusted a signoff receipt that named just finish, and a hand can write such a receipt. It now runs just finish again on the clean, signed HEAD, after acceptance and before the tag. The release test proves that a forged receipt on a repository that finish refuses pushes no tag. Its releases that must pass now commit filled forms and a readiness review first. Closes #419. Co-Authored-By: Claude Opus 5.5 --- scripts/template-finish-test.sh | 4 ++-- scripts/template-ledger-builds-test.sh | 10 ++++++--- scripts/template-release-helpers.sh | 14 ++++++++++++ scripts/template-release-test.sh | 24 +++++++++++++++++++- sdk/extensionrepo/readiness.go | 31 +++++++++++++++----------- sdk/extensionrepo/release.go | 8 ++++++- 6 files changed, 71 insertions(+), 20 deletions(-) diff --git a/scripts/template-finish-test.sh b/scripts/template-finish-test.sh index dab71a5b..33afbf68 100644 --- a/scripts/template-finish-test.sh +++ b/scripts/template-finish-test.sh @@ -122,9 +122,9 @@ fct_test_finish() { fct_finish 1 fct_finish_said 'readiness.md:13: this table names the severity warn' fct_finish_said 'readiness.md:14: this table names the severity block' - fct_finish_said 'readiness.md:19: this fenced code names the severity warn' + fct_finish_said 'readiness.md:19: this fenced code names the severity `warn`' for number in 24 29 33 36; do - fct_finish_said "readiness.md:${number}: this line names the severity" + fct_finish_said "readiness.md:${number}: this line names the" done # Finish reads Commit:, Reviewer:, and Verdict: only in the first lines, never in fenced code (#413). diff --git a/scripts/template-ledger-builds-test.sh b/scripts/template-ledger-builds-test.sh index 86e6fdf0..3d996f9a 100644 --- a/scripts/template-ledger-builds-test.sh +++ b/scripts/template-ledger-builds-test.sh @@ -39,11 +39,15 @@ fct_test_ledger_builds() { fct_finish 1 fct_finish_said 'this line names the word severity outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity' fct_finish_said 'this table names the severity advise' - # A warn in prose beside an advise, or in a table of a part after ## Findings, fails (#416). - fct_readiness "${head}" Ready "${valid}"$'\n\nThis is a warn.\n\n## Notes\n\n| Long lines kept | `warn` |' 'agent B' + # A warn in code beside an advise, or in a table of a part after ## Findings, fails (#416). + fct_readiness "${head}" Ready "${valid}"$'\n\nThis is a `warn`.\n\n## Notes\n\n| Long lines kept | warn |' 'agent B' fct_finish 1 - fct_finish_said 'this line names the severity warn outside the line - **Severity**: of a finding' + fct_finish_said 'this line names the severity `warn` in code, outside the line - **Severity**: of a finding' fct_finish_said 'this table names the severity warn' + # Finish reads a severity only in the line - **Severity**:, so the words + # block and warn pass in a title, in evidence, and in prose (#419). + fct_readiness "${head}" Ready "${valid}"$'\n- **Evidence**: the code block in README.md\n\nThe CLI does not warn when the input is empty.\n\n### R3-F1 Large inputs block the terminal\n\n- **Severity**: `advise`\n\n## Notes\n\nA long line can block a pager.' 'agent B' + fct_finish 0 # Finish reads the header in the first three lines, after a byte order mark (#416). fct_readiness "${head}" Ready "${valid}" 'agent B' printf '# Readiness review\n' | cat - "${finish}/readiness.md" >"${scratch}/readiness.md" diff --git a/scripts/template-release-helpers.sh b/scripts/template-release-helpers.sh index 25457e6a..86b9eecc 100644 --- a/scripts/template-release-helpers.sh +++ b/scripts/template-release-helpers.sh @@ -40,3 +40,17 @@ fct_release_refused_receipts() { fi cp "${scratch}/valid-signoff.json" "${receipt}" } + +# fct_release_review commits a readiness review of HEAD of the repository $1 +# that passes just finish, so the release that runs just finish again goes on +# (#419). The round ends its build, so the next review starts at new steps of +# fct_steps. The caller keeps the earlier rounds in ledger_earlier. +fct_release_review() { + # shellcheck disable=SC2034 # fct_readiness writes readiness.md in finish. + local finish="${1}" head + head="$(git -C "${1}" rev-parse HEAD)" + fct_readiness "${head}" Ready None. + git -C "${1}" add readiness.md + git -C "${1}" commit -qm 'readiness review' + ledger_earlier="${ledger_earlier:+${ledger_earlier} }${head}:pass" +} diff --git a/scripts/template-release-test.sh b/scripts/template-release-test.sh index 0ddaad6a..144a9366 100644 --- a/scripts/template-release-test.sh +++ b/scripts/template-release-test.sh @@ -7,7 +7,7 @@ source "${BASH_SOURCE[0]%/*}/template-release-helpers.sh" source "${BASH_SOURCE[0]%/*}/template-release-url-test.sh" fct_test_release() { - local repo="${1}" scratch="${2}" slug="${3}" extension_root="${4}" sha previous receipt dir status out_file out_bytes err_file err_bytes i examples command + local repo="${1}" scratch="${2}" slug="${3}" extension_root="${4}" sha previous receipt dir status out_file out_bytes err_file err_bytes i examples command file ledger_earlier="" local remote="${2}/release-origin.git" fixture="${2}/release-fixture/input.md" fct_has "${repo}/AGENTS.md" '## Release' fct_has "${repo}/README.md" "ncly describe ${extension_root} --json" @@ -55,6 +55,23 @@ SCRIPT fi [[ -z "$(git -C "${remote}" tag -l)" ]] || fct_fail 'acceptance refusal pushed a tag' + # A receipt that names just finish proves nothing about the repository. + # Release runs just finish again on the signed commit, before the tag (#419). + status=0 + fct_release_attempt "${repo}" "${scratch}" v0.1.0 || status=$? + if [[ "${status}" == 0 ]] || ! grep -qF 'finish exited 1' "${scratch}/release.out"; then + fct_fail 'release trusts a receipt that names just finish, on a repository that just finish refuses' + fi + [[ -z "$(git -C "${remote}" tag -l)" ]] || fct_fail 'a release that just finish refuses pushed a tag' + + # The releases below pass just finish: the forms are filled, and an independent review passes. + fct_steps "${repo}" + for file in vision.md spec.md qa.md; do + grep -v '^Fill:' "${repo}/${file}" >"${scratch}/form.md" + mv "${scratch}/form.md" "${repo}/${file}" + done + git -C "${repo}" add vision.md spec.md qa.md + # Economy selects modes even when the first example already sets them. examples="$(grep -m1 '^examples = ' "${repo}/manifest.toml")" command="$(sed -n 's/^examples = \["ncly \(.*\) notes.md".*$/\1/p' "${repo}/manifest.toml" | head -n 1)" @@ -67,6 +84,7 @@ SCRIPT printf '\nUnpushed change\n' >>"${repo}/README.md" git -C "${repo}" add README.md git -C "${repo}" commit -qm 'unpushed release input' + fct_release_review "${repo}" fct_release_receipt "${repo}" "${scratch}" status=0 fct_release_attempt "${repo}" "${scratch}" v0.1.0 || status=$? @@ -81,10 +99,12 @@ SCRIPT previous="$(git -C "${repo}" rev-parse HEAD)" # D095: a new tag in prose cannot repair an install block pinned to the previous tag. + fct_steps "${repo}" fct_edit "${repo}/README.md" "git clone --branch TAG https://example.com/ncly-${slug} ncly-${slug}" "git clone --branch v0.1.0 https://example.com/ncly-${slug} ncly-${slug}" printf '\nThe current release is v0.1.1.\n' >>"${repo}/README.md" git -C "${repo}" add README.md git -C "${repo}" commit -qm 'pin the README install to the previous tag' + fct_release_review "${repo}" git -C "${repo}" push -q origin HEAD:main fct_release_receipt "${repo}" "${scratch}" status=0 @@ -94,9 +114,11 @@ SCRIPT fi printf 'RED D095: previous-tag README rejected after publication, exit %s\n' "${status}" + fct_steps "${repo}" fct_edit "${repo}/README.md" "git clone --branch v0.1.0 https://example.com/ncly-${slug} ncly-${slug}" "git clone --branch TAG https://example.com/ncly-${slug} ncly-${slug}" git -C "${repo}" add README.md git -C "${repo}" commit -qm 'install the selected release tag' + fct_release_review "${repo}" git -C "${repo}" push -q origin HEAD:main fct_release_receipt "${repo}" "${scratch}" sha="$(git -C "${repo}" rev-parse HEAD)" diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index cd3c5b74..f9b62fbd 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -24,9 +24,12 @@ var ( readinessHeading = regexp.MustCompile(`^(#{1,6})\s+(.+?)\s*$`) readinessPart = regexp.MustCompile(`^\s*[-*+]\s+\*\*(Severity|State|Spec|Commit|Reason|Limit)(?:\*\*:|:\*\*)\s*(.*?)\s*$`) readinessID = regexp.MustCompile(`^(R([1-9][0-9]*)-F[1-9][0-9]*)\b`) - // readinessSeverity finds a severity in a form that finish cannot grade, - // such as prose, code, or a table, anywhere after the header (#413, #416). - readinessSeverity = regexp.MustCompile(`(?i)\b(block|warn|advise)\b`) + // readinessSeverity finds a severity in code, a form that finish cannot + // grade, anywhere after the header (#413, #416). The plain words pass, so + // a title or prose can say that a call blocks or warns (#419). + readinessSeverity = regexp.MustCompile("(?i)`(block|warn|advise)`") + // readinessCell finds a severity alone in a cell of a table (#416). + readinessCell = regexp.MustCompile("(?i)\\|\\s*`?(block|warn|advise)`?\\s*(?:\\||$)") // readinessWord finds the word severity under ## Findings (#413). readinessWord = regexp.MustCompile(`(?i)\bseverity\b`) ) @@ -34,16 +37,17 @@ var ( // severityCause names why a line that names a severity fails finish, so the // reviewer sees the word or the table that caused it (#416). func severityCause(text, fence string) string { - const fix = "Fix: grade each finding only in its line - **Severity**:, and rephrase this line without " - switch m := readinessSeverity.FindString(text); { - case fence != "": - return fmt.Sprintf("this fenced code names the severity %s. %sthe word %s", strings.ToLower(m), fix, strings.ToLower(m)) - case strings.HasPrefix(strings.TrimSpace(text), "|") && m != "": - return fmt.Sprintf("this table names the severity %s. Fix: remove the table, because each finding gives its severity in its line - **Severity**:", strings.ToLower(m)) - case m != "": - return fmt.Sprintf("this line names the severity %s outside the line - **Severity**: of a finding. %sthe word %s", strings.ToLower(m), fix, strings.ToLower(m)) + const fix = "Fix: grade each finding only in its line - **Severity**:, and write " + code, cell := readinessSeverity.FindStringSubmatch(text), readinessCell.FindStringSubmatch(text) + switch { + case cell != nil && strings.HasPrefix(strings.TrimSpace(text), "|"): + return fmt.Sprintf("this table names the severity %s. Fix: remove the table, because each finding gives its severity in its line - **Severity**:", strings.ToLower(cell[1])) + case code != nil && fence != "": + return fmt.Sprintf("this fenced code names the severity `%s`. %sthe word %s in other text without backticks", strings.ToLower(code[1]), fix, strings.ToLower(code[1])) + case code != nil: + return fmt.Sprintf("this line names the severity `%s` in code, outside the line - **Severity**: of a finding. %sthe word %s in other text without backticks", strings.ToLower(code[1]), fix, strings.ToLower(code[1])) } - return "this line names the word severity outside the line - **Severity**: of a finding. " + fix + "the word severity" + return "this line names the word severity outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity" } // readinessFindings reads the findings of readiness.md, outside fenced code. @@ -69,7 +73,8 @@ func readinessFindings(readiness string) ([]finding, []string) { bullet := part > 0 && fence == "" && m != nil && m[1] == "Severity" // The first three lines are the header. A severity elsewhere fails, // also under another part after ## Findings (#416). - if i >= 3 && !bullet && (readinessSeverity.MatchString(text) || part > 0 && !section && readinessWord.MatchString(text)) { + if i >= 3 && !bullet && (readinessSeverity.MatchString(text) || readinessCell.MatchString(text) && strings.HasPrefix(trimmed, "|") || + part > 0 && !section && readinessWord.MatchString(text)) { problems = append(problems, fmt.Sprintf("readiness.md:%d: %s", i+1, severityCause(text, fence))) } switch { diff --git a/sdk/extensionrepo/release.go b/sdk/extensionrepo/release.go index 66dd9e13..87f3fa09 100644 --- a/sdk/extensionrepo/release.go +++ b/sdk/extensionrepo/release.go @@ -75,7 +75,13 @@ func (r *releaseReceipt) publish(ctx context.Context, env []string, root string, if err := r.require(ctx, env, root, "acceptance", acceptance...); err != nil { return err } - // Acceptance cannot change the source that the receipt signed. + // A receipt that names just finish proves nothing about the repository, + // because a hand can write it. So the gate runs again on the signed + // commit, before the tag (#419). + if err := r.require(ctx, env, root, "finish", "just", "finish"); err != nil { + return err + } + // Acceptance and finish cannot change the source that the receipt signed. if err := releasePushed(ctx, env, root, r.SHA); err != nil { return err } From 5bed3fb7139e37fe06b184a1456ee123dc109c77 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 19:50:31 -0400 Subject: [PATCH 28/61] =?UTF-8?q?=E2=9C=A8=20feat:=20template:=20the=20led?= =?UTF-8?q?ger=20records=20the=20first=20failing=20run,=20and=20Open=20lin?= =?UTF-8?q?es=20names=20each=20open=20finding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In round 4 of the day-1 trials, two of three agents committed the scenario before the handler but never ran it. Each spec.md also said None. under ## Open lines while the ledger recorded open findings. readiness.md now holds ## First failing run, with the command that ran the first scenario at its commit, its exit status, and the assertion that failed. Finish fails when the part, the command, or the assertion is missing, or when the exit status is 0 or no number. A finding with the state open now needs a line under ## Open lines of spec.md that names its ID, so spec.md and the ledger agree on the open work. Part of #420. Co-Authored-By: Claude Opus 5.5 --- scripts/template-finish-helpers.sh | 6 +++- scripts/template-ledger-run-test.sh | 34 ++++++++++++++++++ scripts/template-ledger-test.sh | 4 +++ sdk/extensionrepo/ledger.go | 1 + sdk/extensionrepo/ledger_findings.go | 15 ++++++++ sdk/extensionrepo/ledger_run.go | 53 ++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 scripts/template-ledger-run-test.sh create mode 100644 sdk/extensionrepo/ledger_run.go diff --git a/scripts/template-finish-helpers.sh b/scripts/template-finish-helpers.sh index 045ba681..166c2fb4 100644 --- a/scripts/template-finish-helpers.sh +++ b/scripts/template-finish-helpers.sh @@ -58,9 +58,12 @@ fct_step() { # reviewed the heads $2, in order. A head written :pass gives its round # the Finish pass, which ends its build. The newest commits of fct_steps give # the order of round 1 of the last build, and its handler is in the first -# head of that build (#417). +# head of that build (#417). ledger_run holds the part ## First failing run. +# Unset, it records a run that failed on an assertion. Empty, it leaves the +# part out (#420). fct_ledger() { local heads first i head finish_cell handler="" + local run=$'- **Command**: `go test ./... -run TestExtension/inspect`\n- **Exit**: 1\n- **Assertion**: `stdout \'"lines":3\'` of testdata/script/inspect.txtar found no match' read -r -a heads <<<"${2}" first="$(git -C "${1}" rev-list --max-parents=0 HEAD)" printf '\n## Rounds\n\n| Round | Head | Change review | Change commits | Readiness review | Finish |\n|---|---|---|---|---|---|\n' @@ -73,4 +76,5 @@ fct_ledger() { done printf '\n## Order of round 1\n\n| Step | Commit |\n|---|---|\n' printf '| %s | %s |\n' Intake "$(fct_step "${1}" intake)" Spec "$(fct_step "${1}" spec)" Scenario "$(fct_step "${1}" scenario)" Handler "${handler}" + [[ -z "${ledger_run-${run}}" ]] || printf '\n## First failing run\n\n%s\n' "${ledger_run-${run}}" } diff --git a/scripts/template-ledger-run-test.sh b/scripts/template-ledger-run-test.sh new file mode 100644 index 00000000..e71f1fa8 --- /dev/null +++ b/scripts/template-ledger-run-test.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Sourced by template-ledger-test.sh. fct_test_ledger calls fct_test_ledger_run +# at the end of round 3, with its repository and its fixture variables. It +# proves the record of the first failing run of round 1, and that ## Open +# lines names each open finding (#420). Markdown backticks in the fixture +# text are literal, and fct_readiness reads ledger_earlier and ledger_run. +# shellcheck disable=SC2016,SC2034,SC2154 + +fct_test_ledger_run() { + git -C "${finish}" checkout -q readiness.md spec.md + # spec.md and the ledger agree on the open work. + fct_readiness "${head}" 'Ready with stated limits' "${advise}${declined}"$'\n\n'"${warn}"$'\n- **State**: `open`\n- **Spec**: `long-lines`' 'agent B' + fct_finish 1 + fct_finish_said 'spec.md: ## Open lines does not name the open finding R2-F1 of readiness.md' + fct_edit "${finish}/spec.md" 'None.' $'| ID | Command | Part | Case | Code |\n|---|---|---|---|---|\n| `long-lines` | inspect | Failures | R2-F1: a line over 1 MB is cut | `RUNTIME` |' + fct_finish 1 + fct_finish_said 'spec.md: ## Open lines does not name the open finding' absent + git -C "${finish}" checkout -q spec.md + + # Round 1 records the run of its first scenario: the command, a failing + # exit status, and the assertion that failed (#420). + ledger_run='' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said 'readiness.md: it holds no part ## First failing run' + ledger_run=$'- **Command**: `go test ./...`\n- **Exit**: 0\n- **Assertion**:' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said 'the first run of the scenario exited 0' + fct_finish_said 'names no assertion that failed' + ledger_run=$'- **Exit**: one' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said 'names no command' + fct_finish_said 'the exit status "one" of the first run' + git -C "${finish}" checkout -q readiness.md +} diff --git a/scripts/template-ledger-test.sh b/scripts/template-ledger-test.sh index 1a92df69..edc0e4bf 100644 --- a/scripts/template-ledger-test.sh +++ b/scripts/template-ledger-test.sh @@ -6,6 +6,9 @@ # ledger_earlier. # shellcheck disable=SC2016,SC2034 +# shellcheck source=scripts/template-ledger-run-test.sh +source "${BASH_SOURCE[0]%/*}/template-ledger-run-test.sh" + fct_test_ledger() { local finish="${2}/ledger" scratch="${2}" file first intake round1 review1 round2 review2 head stray valid ledger_earlier="" local advise=$'### R1-F1 The count could name the longest line\n\n- **Severity**: `advise`' @@ -145,5 +148,6 @@ fct_test_ledger() { fct_finish 1 fct_finish_said 'readiness.md: it holds no round under ## Rounds' + fct_test_ledger_run fct_test_ledger_builds } diff --git a/sdk/extensionrepo/ledger.go b/sdk/extensionrepo/ledger.go index 7b81e531..9c15ca02 100644 --- a/sdk/extensionrepo/ledger.go +++ b/sdk/extensionrepo/ledger.go @@ -57,6 +57,7 @@ func ledgerProblems(readiness, reviewed string, findings []finding, spec string, } reviews := committedReviews() problems = append(problems, orderProblems(order, ended)...) + problems = append(problems, firstRunProblems(readiness)...) problems = append(problems, stateProblems(findings, len(rounds), spec, limits)...) problems = append(problems, passProblems(rounds, reviews)...) return append(problems, historyProblems(findings, rounds, reviews)...) diff --git a/sdk/extensionrepo/ledger_findings.go b/sdk/extensionrepo/ledger_findings.go index 8a5ca9a2..097b9035 100644 --- a/sdk/extensionrepo/ledger_findings.go +++ b/sdk/extensionrepo/ledger_findings.go @@ -2,8 +2,10 @@ package extensionrepo import ( "fmt" + "regexp" "slices" "strconv" + "strings" ) // stateProblems checks the state of each finding in the build of n rounds. @@ -15,6 +17,13 @@ func stateProblems(findings []finding, n int, spec string, limits []limit) []str proven = append(proven, l.ID) } } + lines := strings.Split(spec, "\n") + var open []string + for _, l := range SpecLines(spec) { + if l.Part == "Open lines" && (l.Kind == SpecRow || l.Kind == SpecBullet) { + open = append(open, lines[l.Number-1]) + } + } var problems []string seen := map[string]bool{} for _, f := range findings { @@ -39,6 +48,12 @@ func stateProblems(findings []finding, n int, spec string, limits []limit) []str } case "open": problems = append(problems, fmt.Sprintf("%sthe finding %s is open, even if ## Open lines holds None. Fix: prove its spec ID in a round, or ask the user to accept it as a limit", at, f.id)) + // spec.md and the ledger give one state of the open work (#420). + named := regexp.MustCompile(`\b` + regexp.QuoteMeta(f.id) + `\b`) + if !slices.ContainsFunc(open, named.MatchString) { + problems = append(problems, fmt.Sprintf("spec.md: ## Open lines does not name the open finding %s of readiness.md. "+ + "Fix: list it under ## Open lines by its ID, such as in the Case of its row, so spec.md and the ledger agree on the open work", f.id)) + } case "fixed": if commit, ok := ledgerCommit(f.commit); !ok || commit == "" { problems = append(problems, at+"this fixed finding names no commit of HEAD. Fix: name the commit that fixed it in a line - **Commit**: ") diff --git a/sdk/extensionrepo/ledger_run.go b/sdk/extensionrepo/ledger_run.go new file mode 100644 index 00000000..fe4512cf --- /dev/null +++ b/sdk/extensionrepo/ledger_run.go @@ -0,0 +1,53 @@ +package extensionrepo + +import ( + "fmt" + "regexp" + "strconv" + "strings" +) + +// firstRun finds a line of ## First failing run. That part records the run of +// the first scenario of round 1 at its commit, before the handler (#420). +var firstRun = regexp.MustCompile(`^\s*[-*+]\s+\*\*(Command|Exit|Assertion)\*\*:\s*(.*?)\s*$`) + +// firstRunProblems checks the record of the first failing run of the current +// build. It needs the command, an exit status that is not 0, and the +// assertion that failed. The reviewer judges whether that assertion proves +// the missing behavior, so a compile error is no proof. +func firstRunProblems(readiness string) []string { + const fix = "Fix: run the first scenario at its commit, before the handler, and record that run under ## First failing run " + + "in the lines - **Command**:, - **Exit**:, and - **Assertion**:" + parts, at, found := map[string]string{}, 0, false + for i, text := range strings.Split(readiness, "\n") { + if h := readinessHeading.FindStringSubmatch(text); h != nil && len(h[1]) <= 2 { + found = found || h[2] == "First failing run" + at = 0 + if h[2] == "First failing run" { + at = i + 1 + } + continue + } + if m := firstRun.FindStringSubmatch(text); at > 0 && m != nil { + parts[m[1]] = strings.Trim(m[2], "`") + } + } + if !found { + return []string{"readiness.md: it holds no part ## First failing run, so no record shows that the first scenario of round 1 failed before the handler. " + fix} + } + var problems []string + if parts["Command"] == "" { + problems = append(problems, "readiness.md: ## First failing run names no command. "+fix) + } + switch exit, err := strconv.Atoi(parts["Exit"]); { + case err != nil: + problems = append(problems, fmt.Sprintf("readiness.md: the exit status %q of the first run is no number. %s", parts["Exit"], fix)) + case exit == 0: + problems = append(problems, "readiness.md: the first run of the scenario exited 0, so it did not fail. "+fix) + } + if strings.TrimSpace(strings.TrimSuffix(parts["Assertion"], ".")) == "" { + problems = append(problems, "readiness.md: ## First failing run names no assertion that failed. "+ + "A compile error is no failed assertion, because it proves no missing behavior. "+fix) + } + return problems +} From 591d25a543e58bbbd8370748c1c71b29bab6b2df Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 19:51:09 -0400 Subject: [PATCH 29/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20template?= =?UTF-8?q?:=20move=20the=20intake=20of=20the=20skill=20to=20its=20own=20f?= =?UTF-8?q?ile?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The skill of the template is near the token limit of a file, and the next commit adds the evidence of the intake. The parts "Intake of a script" and "Intake of a program" move, unchanged, to intake.md, which SKILL.md links. Co-Authored-By: Claude Opus 5.5 --- .../.agents/skills/nicely-extension/SKILL.md | 22 ++-------------- .../.agents/skills/nicely-extension/intake.md | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 template/.agents/skills/nicely-extension/intake.md diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index 0837803e..82dad1ce 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -14,27 +14,9 @@ Read "Extension identity" in the contract for the slug and command ownership. The contract holds every rule. `just contract` prints its folder at the Nicely version that `go.mod` pins. Read "What a great extension does" in its `extensions.md` first. Then read only the parts that it links. -## Intake of a script +## Intake -Get these facts before you write code. Read the source of the script, and ask its author what the source does not show. Do not run the script to learn what it does, because even its `--help` can write or bill. - -- The program and its entry point, such as `python3 tool.py`, with its source and revision, for `vision.md` -- Its inputs: the arguments, stdin, files, environment variables, and the folder that it reads -- Its result: what proves that the work happened, on which stream, and in which format -- Its effects: each write, network call, and paid request, also in a dry run or in `--help` -- Its keys and logins, and where it reads them -- How long it runs, what it starts in the background, and whether it can ask a question -- Its exit codes, and what a repeat of the same call does - -Write each fact that nobody can give as an open line of `spec.md`. - -## Intake of a program - -A third-party program, such as an encoder, has no author to ask. -Get the facts of the script intake from its documentation and its manual page. -Also read its exit codes, its limits, and its known failure modes, such as the open issues of its tracker. -Do not run a command that writes or bills to learn its behavior. -Write each fact that the documentation does not give as an open line of `spec.md`. +Read [intake.md](intake.md) before you write code. It gives the facts to get from a script or from a program. ## Choice of operations diff --git a/template/.agents/skills/nicely-extension/intake.md b/template/.agents/skills/nicely-extension/intake.md new file mode 100644 index 00000000..c22c2967 --- /dev/null +++ b/template/.agents/skills/nicely-extension/intake.md @@ -0,0 +1,25 @@ +# Intake + +This file says which facts the intake of a command gets, before any code. + +## Intake of a script + +Get these facts before you write code. Read the source of the script, and ask its author what the source does not show. Do not run the script to learn what it does, because even its `--help` can write or bill. + +- The program and its entry point, such as `python3 tool.py`, with its source and revision, for `vision.md` +- Its inputs: the arguments, stdin, files, environment variables, and the folder that it reads +- Its result: what proves that the work happened, on which stream, and in which format +- Its effects: each write, network call, and paid request, also in a dry run or in `--help` +- Its keys and logins, and where it reads them +- How long it runs, what it starts in the background, and whether it can ask a question +- Its exit codes, and what a repeat of the same call does + +Write each fact that nobody can give as an open line of `spec.md`. + +## Intake of a program + +A third-party program, such as an encoder, has no author to ask. +Get the facts of the script intake from its documentation and its manual page. +Also read its exit codes, its limits, and its known failure modes, such as the open issues of its tracker. +Do not run a command that writes or bills to learn its behavior. +Write each fact that the documentation does not give as an open line of `spec.md`. From 7270c2871e157323e078b8e134d0bc0a15c5c08d Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 19:51:54 -0400 Subject: [PATCH 30/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20template:=20the=20?= =?UTF-8?q?intake=20shows=20what=20was=20read,=20each=20generic=20case=20g?= =?UTF-8?q?ets=20a=20test=20file,=20and=20a=20broken=20promise=20is=20a=20?= =?UTF-8?q?warn?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 4 of the day-1 trials found gaps in 2 or 3 of 3 extensions. This commit writes the rules of #420 that no check enforces, and the docs of the checks of the commits before it: - intake.md: the intake records the source, its version, and the part that the agent read, with each experiment apart. A link alone is not evidence of a read - generic-cases.md, test-kit.md, and acceptance.sh: each case of the Output group that applies gets a test file that the test makes. A block that a reader needs keeps its content and loses its private text. A declaration that changes the meaning gets a value that is not the default, and the reference applies it - readiness-review.md: a broken promise is at least warn at its first review, with three calibration examples. The reviewer of round 1 checks the intake and the first failing run, and refuses a compile error - SKILL.md: round 1 records the first failing run, tests make their own files, and four rules help a build reach "Done when" in 5 rounds - finish-gate.md, extensions.md, and rule-index.md: the checks of #419 and #420, and "Prose only" for each other new rule Closes #420. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/extensions.md | 3 +- docs/north-star/contract/finish-gate.md | 15 +++++++++- docs/north-star/contract/generic-cases.md | 6 ++-- docs/north-star/contract/test-kit.md | 10 +++++++ .../.agents/skills/nicely-extension/SKILL.md | 22 ++++++++++++--- .../.agents/skills/nicely-extension/intake.md | 11 ++++++++ .../nicely-extension/readiness-review.md | 28 +++++++++++++++++-- template/AGENTS.md.jinja | 2 +- template/AGENTS/rule-index.md.jinja | 12 ++++++++ template/acceptance.sh.jinja | 10 +++++++ template/spec.md.jinja | 2 +- template/vision.md.jinja | 14 ++++++---- 12 files changed, 117 insertions(+), 18 deletions(-) diff --git a/docs/north-star/contract/extensions.md b/docs/north-star/contract/extensions.md index 5844bba9..fe583319 100644 --- a/docs/north-star/contract/extensions.md +++ b/docs/north-star/contract/extensions.md @@ -24,7 +24,8 @@ An author releases a version with `just release vX.Y.Z [fixture]`. The SDK requires a clean HEAD equal to the fetched default branch of `origin`. A successful signoff receipt must name that SHA, with exit 0 for every check and signoff. Its checks must hold `just finish`, so a receipt with only `just check` never releases. -Release runs `just acceptance [fixture]`, then creates and pushes an annotated tag whose message holds that signoff receipt. +Release runs `just acceptance [fixture]`, then `just finish` again on that clean HEAD, because a hand can write a receipt (#419). +Then it creates and pushes an annotated tag whose message holds that signoff receipt. It executes the fenced `sh` block under `## Install` in `README.md`, with `TAG` replaced by the new tag. The install uses scratch HOME and XDG folders, without the caller's keys or installed extensions. The clone's HEAD must equal the tagged SHA, even when another README paragraph names the new tag. diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index f6f85f98..99879ff4 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -12,7 +12,10 @@ This part says what `just finish` checks in an extension repository, as [Extensi - A `readiness.md` whose first three lines hold no `Commit: `, no `Reviewer: `, or no verdict `Ready` or `Ready with stated limits`. A byte order mark before them is ignored - A reviewed commit that differs from the working tree outside `readiness.md`, and outside the part **Limits.** of `spec.md`. So the user accepts a limit after a review, with no new round - A finding of `readiness.md` without its own `###` heading under `## Findings`, or without one severity: `block`, `warn`, or `advise` -- A severity after the first three lines, outside the line `- **Severity**:` of a finding, such as in a table, in fenced code, or in prose. Under `## Findings`, the word severity fails too. The message names the word or the table that caused it +- A severity after the first three lines, outside the line `- **Severity**:` of a finding, in a form that grades. Such a form is code, such as `` `warn` ``, also in fenced code, or a cell of a table that holds only a severity. Under `## Findings`, the word severity fails too. The message names the word or the table that caused it + +Finish reads a severity only in the line `- **Severity**:` of a finding. +The plain words block, warn, and advise pass in a title, in evidence, and in prose, such as `### R1-F1 Large inputs block the terminal` (#419). - `Ready` with a `block` or `warn` finding that is not `fixed`, or with a limit in `spec.md` - `Ready with stated limits` with a `block` finding that is not `fixed`, or with a `warn` that is neither `fixed` nor an accepted `limit` @@ -34,6 +37,14 @@ A round before the last whose Finish is `pass` ends a build. The next round star Under `## Order of round 1`, a table with the columns Step and Commit names the commit of each step of the current build: Intake, Spec, Scenario, and Handler. +Under `## First failing run`, three lines record the run of the first scenario of the current build at its commit, before the handler (#420): + +- `- **Command**:` the command that ran the scenario +- `- **Exit**:` its exit status, which is not 0 +- `- **Assertion**:` the assertion of the scenario that failed + +The reviewer refuses a failure that is only a compile error, because it does not prove the missing behavior. + Each finding under `## Findings` starts its heading with a stable ID, such as `### R2-F3`. `R2` is the round that first reported it, and the ID stays in later rounds. The reviewer of a later round gives each earlier finding one state, in a line `- **State**:`: @@ -57,8 +68,10 @@ A finding of the last round can have no state yet. - A commit of a round or of a fixed finding that is not HEAD or an ancestor of HEAD - A last round whose Head is not the `Commit:` of `readiness.md` - A step of round 1 that is missing, out of order, or in the commit of the step before it +- No part `## First failing run`, or one without its command, without an exit status other than 0, or without its assertion - A finding without an ID, a finding of an earlier round without a state, or a state without its parts - A finding with the state `open`, even when `## Open lines` holds `None.` +- A finding with the state `open` whose ID no line under `## Open lines` of `spec.md` names. So `spec.md` and the ledger agree on the open work (#420) - A `warn` or `block` finding with the state `declined` - A finding of an earlier committed `readiness.md` that is missing, or whose severity went down. A finding without an ID keeps its title - A committed review whose `Commit:` is the Head of no round, or a round before the last without a committed review of its Head diff --git a/docs/north-star/contract/generic-cases.md b/docs/north-star/contract/generic-cases.md index 5d20df50..707e394c 100644 --- a/docs/north-star/contract/generic-cases.md +++ b/docs/north-star/contract/generic-cases.md @@ -8,6 +8,8 @@ Each case that can occur becomes one line, with its spec ID, as [Spec IDs](spec- The line states the outcome of the case, such as a refusal with its code. The safe outcome below is the default. Another outcome is a limit, and only the user can accept it. A case that cannot occur for the command needs no line. +A case name is not a test. For each case of the Output group that applies to the command, `just acceptance` checks a test file of the extension's own domain. +The test makes that file, and an independent reader proves the safe outcome on it, as [Test kit](test-kit.md#acceptance-test-files) says. An error that names the input gives its path or its item, so an agent knows which call failed. | Group | Case | Example | Safe outcome | @@ -25,8 +27,8 @@ An error that names the input gives its path or its item, so an agent knows whic | Output | Appears during the run, without `--force` | Another process writes `notes.txt` after the check and before the write | The command never replaces that file. It publishes with a create that fails on an existing file, such as a hard link. When the file system cannot create the output without a replace, the command fails. It never uses a check and then a rename that replaces | | Output | Temporary file | A link `notes.txt.tmp` to `notes.md`, made before the run | The command creates its temporary file with exclusive create, and with a name that no other process can predict. It never writes through a link at that path | | Output | Keeps private data of the input | `notes.md` starts with the line `author: Ana Lima`, and `notes.txt` keeps that name | The command removes the private data, or the user accepts a limit | -| Output | Keeps data that a reader needs | `notes.md` starts with a header that declares its text encoding and names its author | The command keeps the data that the reader needs to show the content correctly, and removes the private text. Otherwise, the user accepts a limit | -| Output | Loses data that changes the meaning | `notes.md` declares the encoding `latin-1`, and `notes.txt` keeps its bytes without that line | The output keeps the declaration, such as a text encoding or a unit, or the command applies it to the content | +| Output | Keeps data that a reader needs | `notes.md` starts with a valid header that a reader needs, with the line `encoding: latin-1`, and that header also holds the line `author: Ana Lima` | The command keeps the data that the reader needs to show the content correctly, and removes the private text. The test proves both: `Ana Lima` is not in the output, and a reader still shows the content correctly. Otherwise, the user accepts a limit | +| Output | Loses data that changes the meaning | `notes.md` declares the encoding `latin-1`, and `notes.txt` keeps its bytes without that line | The output keeps the declaration, such as a text encoding or a unit, or the command applies it to the content. The intake lists each declaration that changes how a reader shows the content, not only the file types. The test file uses a value that is not the default, and the independent reference applies that declaration before it compares the content | | Interruption | SIGINT or SIGTERM during the work | Ctrl-C while `inspect` counts the lines of a large file | Exit 130 or 143, with an answer that names the input. No partial output remains, the old output stays, and no child process remains | | Interruption | Timeout | `inspect --timeout 1ms` on a large file | The timeout outcome, with an answer that names the input, even when the check of the program uses up the deadline. No partial output remains | | File name | Spaces | `my notes.md` | The command passes the name as one path, unchanged | diff --git a/docs/north-star/contract/test-kit.md b/docs/north-star/contract/test-kit.md index 3d6da4eb..0449ca3d 100644 --- a/docs/north-star/contract/test-kit.md +++ b/docs/north-star/contract/test-kit.md @@ -50,6 +50,16 @@ The kit proves the declared cases. Whether a claimed test proves its line, and w A check that reads only a header or the file type is not sufficient. The check compares the content that this reader gives with an independent reference, such as a file of expected text that the author writes. +### Acceptance test files + +Each case of the Output group of [Generic cases](generic-cases.md) that applies to the command needs a test file (#420). +`acceptance.sh` makes that file in the extension's own domain, and proves the safe outcome on it. Two cases need more: + +- **Keeps data that a reader needs.** The test file holds a valid block that a reader needs, and that block holds private text. The check proves both results: the private text is not in the output, and the output still shows the content correctly +- **Loses data that changes the meaning.** The test file gives each declaration that changes how a reader shows the content a value that is not the default. The independent reference applies that declaration before it compares the content, so a lost declaration fails the check + +The tests make each file that they use. They never copy a file of the user, and the author never searches the home folder of the user for sample files. + Other scenario runners use `NewCoverage`, `Coverage.Observe`, and `Coverage.Missing` for the same answer and coverage checks. `Observe` receives completed calls, including their exit code and both streams, without changing process execution or output. Nicely's runner registers these checks for `agent` and `transcript`, for successful and failed calls. diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index 82dad1ce..6ee0ce1e 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -45,6 +45,12 @@ The lines of Failures and Acceptance in `spec.md` are the acceptance criteria of 3. Give each line a spec ID and a state, as the opening of `spec.md` says: `open`, `proven`, or `limit`. A case that cannot occur for the command needs no line. +A case name is not a test. Each case of the Output group that applies gets a test file in `acceptance.sh`, as "Acceptance test files" of `test-kit.md` in the contract says. + +## Test files + +The tests of the extension use files that the tests make, such as a text file that a scenario or `acceptance.sh` writes. +Do not search the home folder of the user for sample files. Never copy a file of the user into a test. ## Rounds @@ -57,16 +63,17 @@ Each round ends with the independent review, the readiness review, and `just fin Do these steps in this order. Commit each of the first four steps on its own: -1. Record the intake facts under `## Intake` in `vision.md`, with the source and the version of each fact. The facts include the effects, the exit codes, the resource use, and the known failures. A fact that no source gives stays an open line of `spec.md`. +1. Record the intake facts under `## Intake` in `vision.md`, as "Evidence of the intake" in `intake.md` says. The facts include the effects, the exit codes, the resource use, and the known failures. A fact that no source gives stays an open line of `spec.md`. 2. Write every line of the command in `spec.md`, from the generic cases of the contract and the intake. Each line that has no test stays `open`, under `## Open lines`. -3. Write the first scenario at the public command boundary, with its claim `# spec: `. Commit it while it fails. +3. Write the first scenario at the public command boundary, with its claim `# spec: `. Commit it while it fails. Run it at that commit, and keep the command, its exit status, and the assertion that failed. A compile error is no such failure. When the scenario cannot run without the command, commit a handler that does no work in this step. Then the run reaches the assertion. 4. Change the handler until that scenario passes, with "Change a command" in `AGENTS.md`. 5. When no line stays open, write `None.` under `## Open lines`, with a sentence that cites the generic cases. 6. Replace every `Fill:` line of `vision.md`, `spec.md`, and `qa.md`. -7. Fill in `acceptance.sh` from `## Acceptance` in `spec.md`. For a paid command, limit it to the dry run. +7. Fill in `acceptance.sh` from `## Acceptance` in `spec.md`, with a test file for each generic case of the Output group that applies. For a paid command, limit it to the dry run. 8. End the round. Give the commits of steps 1 to 4 to the reviewer, who records them under `## Order of round 1` in the ledger. +Give the reviewer the record of the run of step 3 too. The reviewer checks it, and records it under `## First failing run`. The first round is incomplete by design. The open lines start the next round. ### Each next round @@ -84,11 +91,18 @@ The first round is incomplete by design. The open lines start the next round. 3. Run `just acceptance` on the real input until its independent checks pass. 4. Ask an agent that did not write the round to review the changes of the round. Give it the commits of the round. 5. Ask an agent that did not write the code to run the readiness review. It records the review and the ledger in `readiness.md`. One independent pass can do both reviews. -6. Write each finding that a test can prove as an open line. +6. Write each finding that a test can prove as an open line, which names the finding by its ID. Each finding that the ledger records as `open` keeps its line under `## Open lines`, so `spec.md` and the ledger agree. 7. Run `just finish`. Stop as "Done when" in `AGENTS.md` says. `just finish` fails until the loop is done. 8. When `just finish` fails after round 5, stop, as "Round limit" says. 9. Otherwise, start the next round from the open lines and the open findings. Add its commits to the open pull request, or open a new one. Base a new pull request on the branch of the previous one when that one is not merged. +### Reach "Done when" in 5 rounds + +- Round 1 includes a test file for each generic case that applies +- Ask the first reviewer to test families of cases, such as all the declarations that change the meaning, not one counterexample +- Fix a root cause, such as a parser or a conversion rule, before the next review +- Keep the last rounds for verification, not for one patch per warning + ### Round limit At the limit of 5 rounds, stop. Do not start round 6. diff --git a/template/.agents/skills/nicely-extension/intake.md b/template/.agents/skills/nicely-extension/intake.md index c22c2967..a175c632 100644 --- a/template/.agents/skills/nicely-extension/intake.md +++ b/template/.agents/skills/nicely-extension/intake.md @@ -23,3 +23,14 @@ Get the facts of the script intake from its documentation and its manual page. Also read its exit codes, its limits, and its known failure modes, such as the open issues of its tracker. Do not run a command that writes or bills to learn its behavior. Write each fact that the documentation does not give as an open line of `spec.md`. + +## Evidence of the intake + +The intake records what you read, not only where the facts are (#420): + +- For each fact: the source, its version or revision, and the part that you read. Such a part is a function of the script or a section of the manual page +- The result of each experiment, apart from the documented facts, with its command and what it gave +- Each declaration of the input that changes how a reader shows the content, not only the file types. `generic-cases.md` in the contract says why +- Each fact that no source gives, as an open line of `spec.md` + +A link alone is not evidence of a read. Read the source first. Then run experiments only to confirm what you read. diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 28abc825..392901ba 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -36,18 +36,25 @@ Write each part of a finding as a bullet that starts with its bold name, such as - **Owner**: who can fix it, the author of the extension or the author of the script - **Fix**: the smallest change that removes the problem -Name a severity only in the bullet **Severity** of its finding. -After the first three lines, `just finish` fails on the words block, warn, or advise anywhere else. -Examples are a table, fenced code, and prose. +Name a severity only in the bullet **Severity** of its finding. `just finish` reads no other line as a grade. +After the first three lines, it fails on a severity in code, such as `` `warn` ``, on a severity alone in a cell of a table, and on the word severity under `## Findings`. +The plain words pass in a title, in evidence, and in prose, such as `### R1-F1 Large inputs block the terminal`. A loss that a user can see is `warn` or `block`, never `advise`. Examples are data that the output keeps or changes, and a promise of `spec.md` that does not hold. When in doubt between `warn` and `advise`, use `warn`. +A broken promise of `spec.md` or of the help text is at least `warn` at its first review. This applies also when no data is lost. +Grade it so in the round that first reports it. These examples set the grade: + +- **A message that names a wrong cause.** `inspect` answers that `notes.md` is missing, but the file exists and is not readable. At least `warn`, because the agent fixes the wrong thing +- **A dry run that predicts a result that the run does not give.** The dry run says that the output is `notes.txt`, and the run writes `notes.md.txt`. At least `warn`. `block` when the run replaces a file that the dry run said it keeps +- **An output whose meaning changes.** `notes.txt` drops the line `encoding: latin-1` and keeps the bytes, so a reader shows other letters. At least `warn`. `block` when `spec.md` promises the same text Never lower the severity of a finding in a later round. A `warn` or `block` finding closes only as `fixed` or `limit`. Only an `advise` can be `declined`. A change of `spec.md` that narrows a promise to close a `warn` or `block` finding is a `limit`. Only the user can accept it. `None.` under `## Open lines` does not close a finding that the ledger records as `open`. +Each finding with the state `open` keeps a line under `## Open lines` of `spec.md` that names its ID. `just finish` fails without it. End the review with one verdict. The verdict follows from the findings, and `just finish` checks it: @@ -62,6 +69,15 @@ Commit the limit in `spec.md`, then commit `readiness.md` alone. Its State is `limit`. Accepted records who accepted it, when, and where, such as a review comment. A loss that you report but that the user did not accept is a `warn` finding, not a limit. +## Round 1 + +In round 1, review the order of the work, not only the code: + +- The intake commit comes before the spec commit. The intake gives, for each fact, its source, its version, and the part that the author read, and the result of each experiment apart. A link alone is not evidence of a read +- The record of the first failing run gives the command, its exit status, and the assertion that failed at the scenario commit. Run the command again at that commit when in doubt. Refuse a failure that is only a compile error, because it does not prove the missing behavior +- Each case of the Output group of `generic-cases.md` that applies has a test file that the tests make, and an independent check. A case name alone is not a test +- Test families of cases, such as all the declarations that change the meaning, not one counterexample + ## Ledger After the findings, keep the ledger of the build. Add one row for this round, and keep the rows of earlier rounds: @@ -82,6 +98,12 @@ After the findings, keep the ledger of the build. Add one row for this round, an | Spec | | | Scenario | | | Handler | | + +## First failing run + +- **Command**: +- **Exit**: +- **Assertion**: ``` Change commits names the commits that the change review covered. The readiness review covers Head. diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index 6b347786..a09a0450 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -103,7 +103,7 @@ It never replaces "Done when". `just signoff` runs `just finish` first, so a mer 3. Run `just signoff` for that pushed commit 4. Make sure that "Done when" holds for this commit 5. Merge that exact signed SHA into the default branch -6. Run `just release vX.Y.Z`, with an optional fixture path +6. Run `just release vX.Y.Z`, with an optional fixture path. It runs `just acceptance`, then `just finish` again on that commit, before it makes the tag 7. Read the printed receipt for acceptance, the published install, Economy reservations, hosted checks, and uninstall For release recovery, read "Release and install" in the pinned contract's `extensions.md`. diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index 97bb14bc..0c234484 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -21,6 +21,13 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | A test claims each line of Failures and Acceptance in `spec.md` | [Kit spec IDs][spec] | | One bounded job, no arbitrary argument relay; the skill teaches when | Prose only | | Round 1 records the intake, then the spec, then a failing scenario, then the handler | [Finish gate][finish] checks their commits in the [ledger][ledger] | +| The intake gives, for each fact, its source, its version, and the part read, with each experiment apart; a link alone is not evidence | Prose only | +| Round 1 records the command, the exit status, and the failed assertion of the first scenario run | The [ledger][ledger] checks the record | +| A compile error is no failure of the first scenario | Prose only | +| Each case of the Output group that applies has a test file that the tests make, checked by an independent reader | Prose only | +| The tests make their files; never search the home folder or copy a file of the user | Prose only | +| Round 1 tests each generic case that applies, and the first review tests families of cases | Prose only | +| A root cause is fixed before the next review, and the last rounds verify | Prose only | | Build each command with the `nicely-extension` skill, in rounds from its first pull request | Prose only | | The first round writes every line of `spec.md`; a line without a test stays `open` | Prose only | | Each round ends with an independent review and the readiness review | [Signoff][signoff] runs the [finish gate][finish], which needs a readiness review of the signed commit by a named reviewer | @@ -28,8 +35,12 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | A build has at most 5 rounds | The [ledger][ledger] counts them, and the [finish gate][finish] fails after 5 | | At the limit, stop and report; an answer that lets the agent choose starts no round | Prose only | | A pull request for review, a stop report, and the delivery are three acts; deliver only after "Done when" | [Signoff][signoff] runs the [finish gate][finish], and [release][release] needs a receipt with `just finish` | +| Release runs `just finish` again on the signed commit before the tag | [Release][release] | | The ledger names the head, the reviewers, the commits that each review covered, and the result of `just finish` of each round | [Ledger][ledger] | | Each finding keeps its ID and its highest severity, and gets a state each round; a `warn` or `block` closes only as `fixed` or `limit` | [Ledger findings][ledger-findings], [ledger history][ledger-history] | +| `## Open lines` of `spec.md` names each finding that the ledger records as `open`, by its ID | [Ledger findings][ledger-findings] | +| A severity is read only in the line **Severity** of a finding; the plain words block, warn, and advise pass elsewhere | [Readiness findings][readiness] | +| A broken promise of `spec.md` or of the help text is at least `warn` at its first review | Prose only | | Each round keeps its row and has a committed review; a build ends only at a review that passes | [Ledger history][ledger-history] | | Accepting a limit starts no round; after a review, only **Limits.** of `spec.md` changes | [Finish gate][finish-verdict] | | A change of `spec.md` that narrows a promise to close a `warn` or `block` is a limit that only the user accepts | Prose only | @@ -50,6 +61,7 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". [ledger-findings]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger_findings.go [ledger-history]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger_history.go [finish-verdict]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/readiness_verdict.go +[readiness]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/readiness.go [signoff]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/signoff.go [release]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/release.go [template-test]: https://github.com/pascalandy/nicely/blob/main/scripts/template-test.sh diff --git a/template/acceptance.sh.jinja b/template/acceptance.sh.jinja index 4265b517..0bccd580 100644 --- a/template/acceptance.sh.jinja +++ b/template/acceptance.sh.jinja @@ -79,6 +79,16 @@ grep -qE "\"size_bytes\":$(wc -c <"$input" | tr -d ' ')[,}]" "$scratch/answer" | # spec: lines grep -qE "\"lines\":$(awk 'END { print NR }' "$input")[,}]" "$scratch/answer" || die "the line count differs from awk" +# Fill: for each case of the Output group of generic-cases.md in the contract +# that applies to the command, make a test file of this domain here, and check +# its safe outcome as "Acceptance test files" of test-kit.md says. This script +# makes each file. Never copy a file of the user. Such as: a header that a +# reader needs, with the lines encoding: latin-1 and author: Ana Lima. The +# output loses the name, and a reader still shows the same text. A declaration +# that changes the meaning gets a value that is not the default, and the +# reference applies it before the compare. inspect writes no output, so it has +# no such case. + # A second run checks the Determinism criterion of qa.md. (cd "$work" && ncly "${command[@]}" "$input" --json --no-input) >"$scratch/second" || die "the second run failed" cmp -s "$scratch/answer" "$scratch/second" || die "two runs give different answers: $(cat "$scratch/second")" diff --git a/template/spec.md.jinja b/template/spec.md.jinja index cc358f18..2a2bd569 100644 --- a/template/spec.md.jinja +++ b/template/spec.md.jinja @@ -61,4 +61,4 @@ A round of the skill in `.agents/skills/nicely-extension/` moves an open line in None. -Fill: each line that no test proves yet, in a table with the columns ID, Command, Part, Case, and Code. The test kit does not read this part. +Fill: each line that no test proves yet, in a table with the columns ID, Command, Part, Case, and Code. Name each finding that `readiness.md` records as `open` by its ID, such as at the start of its Case, so `spec.md` and the ledger agree. The test kit does not read this part. diff --git a/template/vision.md.jinja b/template/vision.md.jinja index e8950b84..54c8d71c 100644 --- a/template/vision.md.jinja +++ b/template/vision.md.jinja @@ -16,12 +16,16 @@ Fill: the source and the revision of each program or script that a command wraps ## Intake -Fill: each fact of the intake of each wrapped program, with its source and the version of that source. Give its effects, its exit codes, its resource use, and its known failures. A fact that no source gives is an open line of `spec.md`. +Fill: each fact of the intake of each wrapped program. Give its source, the version of that source, and the part that you read, such as a section of the manual page. A link alone is not evidence of a read. Give its effects, its exit codes, its resource use, and its known failures. Give each declaration of the input that changes how a reader shows the content. Give the result of each experiment apart, under **Experiments.** A fact that no source gives is an open line of `spec.md`. -- Effects: `worker/inspect.py` reads the file once, in chunks of 64 KiB, and writes nothing. Source: `worker/inspect.py` of the template, at the Nicely version that `go.mod` pins -- Exit codes: 0 with one JSON object on stdout, and 2 for a wrong argument count. Source: the same file and version -- Resource use: memory stays near one chunk, whatever the size of the file. Source: the same file and version -- Known failures: Python raises an error and exits 1 on a file that it cannot open. Source: the same file and version +- Effects: `worker/inspect.py` reads the file once, in chunks of 64 KiB, and writes nothing. Source: `worker/inspect.py` of the template, at the Nicely version that `go.mod` pins. Read: the function `main` +- Exit codes: 0 with one JSON object on stdout, and 2 for a wrong argument count. Source: the same file and version. Read: `main` and the last two lines of the file +- Resource use: memory stays near one chunk, whatever the size of the file. Source: the same file and version. Read: the loop of `main` +- Known failures: Python raises an error and exits 1 on a file that it cannot open. Source: the same file and version. Read: the call to `open` in `main`, which nothing catches + +**Experiments.** + +- `python3 worker/inspect.py notes.md` on a file of three lines, the last without a newline, printed `"lines": 3` and exited 0 ## Refuses From 0deb72f80e5731a96522ffbaf364d778a8f9e796 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 20:12:03 -0400 Subject: [PATCH 31/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20the=20f?= =?UTF-8?q?irst=20failing=20run=20names=20its=20scenario=20commit,=20and?= =?UTF-8?q?=20no=20grade=20hides=20outside=20a=20finding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An independent review of the commits before found three gaps: - A record of ## First failing run from an earlier build still passed. The record now names its commit, which must be the Scenario commit of ## Order of round 1. A record in fenced code, or in the form **Name:**, now reads as finish expects - After #419, "Severity: block" in a part after ## Findings, a finding heading under another part, and a cell such as **Block.** passed. Finish now fails on the word severity before a severity in any part, on a finding heading with an ID outside ## Findings, and on a cell that holds only a severity with marks around it. "Each finding names its severity" still passes - Release ran just finish after acceptance, which can be slow. It now runs just finish first Co-Authored-By: Claude Opus 5.5 --- scripts/template-finish-helpers.sh | 8 +++--- scripts/template-finish-test.sh | 2 +- scripts/template-ledger-builds-test.sh | 6 +++++ scripts/template-ledger-run-test.sh | 7 +++++ scripts/template-release-test.sh | 14 +++++----- sdk/extensionrepo/ledger.go | 2 +- sdk/extensionrepo/ledger_run.go | 36 +++++++++++++++++++------- sdk/extensionrepo/readiness.go | 33 ++++++++++++++++++----- sdk/extensionrepo/release.go | 8 +++--- 9 files changed, 84 insertions(+), 32 deletions(-) diff --git a/scripts/template-finish-helpers.sh b/scripts/template-finish-helpers.sh index 166c2fb4..410dc4cf 100644 --- a/scripts/template-finish-helpers.sh +++ b/scripts/template-finish-helpers.sh @@ -59,11 +59,11 @@ fct_step() { # the Finish pass, which ends its build. The newest commits of fct_steps give # the order of round 1 of the last build, and its handler is in the first # head of that build (#417). ledger_run holds the part ## First failing run. -# Unset, it records a run that failed on an assertion. Empty, it leaves the -# part out (#420). +# Unset, it records a run of the newest scenario step that failed on an +# assertion. Empty, it leaves the part out (#420). fct_ledger() { - local heads first i head finish_cell handler="" - local run=$'- **Command**: `go test ./... -run TestExtension/inspect`\n- **Exit**: 1\n- **Assertion**: `stdout \'"lines":3\'` of testdata/script/inspect.txtar found no match' + local heads first i head finish_cell handler="" run + run=$'- **Commit**: '"$(fct_step "${1}" scenario)"$'\n- **Command**: `go test ./... -run TestExtension/inspect`\n- **Exit**: 1\n- **Assertion**: `stdout \'"lines":3\'` of testdata/script/inspect.txtar found no match' read -r -a heads <<<"${2}" first="$(git -C "${1}" rev-list --max-parents=0 HEAD)" printf '\n## Rounds\n\n| Round | Head | Change review | Change commits | Readiness review | Finish |\n|---|---|---|---|---|---|\n' diff --git a/scripts/template-finish-test.sh b/scripts/template-finish-test.sh index 33afbf68..def93617 100644 --- a/scripts/template-finish-test.sh +++ b/scripts/template-finish-test.sh @@ -124,7 +124,7 @@ fct_test_finish() { fct_finish_said 'readiness.md:14: this table names the severity block' fct_finish_said 'readiness.md:19: this fenced code names the severity `warn`' for number in 24 29 33 36; do - fct_finish_said "readiness.md:${number}: this line names the" + fct_finish_said "readiness.md:${number}: this line " done # Finish reads Commit:, Reviewer:, and Verdict: only in the first lines, never in fenced code (#413). diff --git a/scripts/template-ledger-builds-test.sh b/scripts/template-ledger-builds-test.sh index 3d996f9a..d2e69295 100644 --- a/scripts/template-ledger-builds-test.sh +++ b/scripts/template-ledger-builds-test.sh @@ -44,6 +44,12 @@ fct_test_ledger_builds() { fct_finish 1 fct_finish_said 'this line names the severity `warn` in code, outside the line - **Severity**: of a finding' fct_finish_said 'this table names the severity warn' + # A grade in prose, a finding outside ## Findings, or a severity with marks in a cell fails in any part (#420). + fct_readiness "${head}" Ready "${valid}"$'\n\n## Notes\n\n- Long lines are cut. Severity: block.\n\n## Other issues\n\n### R3-F9 Long lines are cut\n\n| R3-F9 | **Block.** |' 'agent B' + fct_finish 1 + fct_finish_said 'this line grades with the severity block outside the line - **Severity**: of a finding' + fct_finish_said 'this finding is outside ## Findings, so finish cannot grade it' + fct_finish_said 'this table names the severity block' # Finish reads a severity only in the line - **Severity**:, so the words # block and warn pass in a title, in evidence, and in prose (#419). fct_readiness "${head}" Ready "${valid}"$'\n- **Evidence**: the code block in README.md\n\nThe CLI does not warn when the input is empty.\n\n### R3-F1 Large inputs block the terminal\n\n- **Severity**: `advise`\n\n## Notes\n\nA long line can block a pager.' 'agent B' diff --git a/scripts/template-ledger-run-test.sh b/scripts/template-ledger-run-test.sh index e71f1fa8..a8493c3a 100644 --- a/scripts/template-ledger-run-test.sh +++ b/scripts/template-ledger-run-test.sh @@ -30,5 +30,12 @@ fct_test_ledger_run() { fct_finish 1 fct_finish_said 'names no command' fct_finish_said 'the exit status "one" of the first run' + # A record of another commit, such as of an earlier build, or in fenced code, records no run of this build. + ledger_run=$'- **Commit**: '"${intake}"$'\n- **Command**: `go test ./...`\n- **Exit**: 1\n- **Assertion**: `stdout 3`' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said "names the commit \"${intake}\", which is not the Scenario commit of ## Order of round 1" + ledger_run=$'```\n- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `go test ./...`\n- **Exit**: 1\n- **Assertion**: `stdout 3`\n```' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said 'names no command' git -C "${finish}" checkout -q readiness.md } diff --git a/scripts/template-release-test.sh b/scripts/template-release-test.sh index 144a9366..abe09309 100644 --- a/scripts/template-release-test.sh +++ b/scripts/template-release-test.sh @@ -48,15 +48,9 @@ SCRIPT fct_fail 'release accepts a dirty tree' fi git -C "${repo}" restore README.md - status=0 - fct_release_attempt "${repo}" "${scratch}" v0.1.0 missing-fixture.md || status=$? - if [[ "${status}" == 0 ]] || ! grep -qF 'acceptance exited' "${scratch}/release.out"; then - fct_fail 'release accepts failed acceptance' - fi - [[ -z "$(git -C "${remote}" tag -l)" ]] || fct_fail 'acceptance refusal pushed a tag' # A receipt that names just finish proves nothing about the repository. - # Release runs just finish again on the signed commit, before the tag (#419). + # Release runs just finish again on the signed commit, before acceptance and the tag (#419). status=0 fct_release_attempt "${repo}" "${scratch}" v0.1.0 || status=$? if [[ "${status}" == 0 ]] || ! grep -qF 'finish exited 1' "${scratch}/release.out"; then @@ -92,6 +86,12 @@ SCRIPT fct_fail 'release accepts an unpushed SHA' fi git -C "${repo}" push -q origin HEAD:main + status=0 + fct_release_attempt "${repo}" "${scratch}" v0.1.0 missing-fixture.md || status=$? + if [[ "${status}" == 0 ]] || ! grep -qF 'acceptance exited' "${scratch}/release.out"; then + fct_fail 'release accepts failed acceptance' + fi + [[ -z "$(git -C "${remote}" tag -l)" ]] || fct_fail 'acceptance refusal pushed a tag' fct_release_attempt "${repo}" "${scratch}" v0.1.0 || fct_fail 'first local release failed' receipt="$(sed -n 's/^release: receipt in //p' "${scratch}/release.out")" PATH="/usr/bin:${PATH}" jq -r '.economy[] | select(.command[1:3] != ["skill", "view"] and .command[1] != "describe" and .command[1] != "doctor") | "default Economy: \(.command | join(" ")), exit \(.exit)"' "${receipt}" diff --git a/sdk/extensionrepo/ledger.go b/sdk/extensionrepo/ledger.go index 9c15ca02..7a543915 100644 --- a/sdk/extensionrepo/ledger.go +++ b/sdk/extensionrepo/ledger.go @@ -57,7 +57,7 @@ func ledgerProblems(readiness, reviewed string, findings []finding, spec string, } reviews := committedReviews() problems = append(problems, orderProblems(order, ended)...) - problems = append(problems, firstRunProblems(readiness)...) + problems = append(problems, firstRunProblems(readiness, order)...) problems = append(problems, stateProblems(findings, len(rounds), spec, limits)...) problems = append(problems, passProblems(rounds, reviews)...) return append(problems, historyProblems(findings, rounds, reviews)...) diff --git a/sdk/extensionrepo/ledger_run.go b/sdk/extensionrepo/ledger_run.go index fe4512cf..cb9a9de9 100644 --- a/sdk/extensionrepo/ledger_run.go +++ b/sdk/extensionrepo/ledger_run.go @@ -7,19 +7,30 @@ import ( "strings" ) -// firstRun finds a line of ## First failing run. That part records the run of -// the first scenario of round 1 at its commit, before the handler (#420). -var firstRun = regexp.MustCompile(`^\s*[-*+]\s+\*\*(Command|Exit|Assertion)\*\*:\s*(.*?)\s*$`) +// firstRun finds a line of ## First failing run, in the bold forms that +// readinessPart reads. That part records the run of the first scenario of +// round 1 at its commit, before the handler (#420). +var firstRun = regexp.MustCompile(`^\s*[-*+]\s+\*\*(Commit|Command|Exit|Assertion)(?:\*\*:|:\*\*)\s*(.*?)\s*$`) // firstRunProblems checks the record of the first failing run of the current -// build. It needs the command, an exit status that is not 0, and the -// assertion that failed. The reviewer judges whether that assertion proves -// the missing behavior, so a compile error is no proof. -func firstRunProblems(readiness string) []string { +// build. It needs the Scenario commit of ## Order of round 1, so a record of +// an earlier build never stands in. It also needs the command, an exit status +// that is not 0, and the assertion that failed. The reviewer judges whether +// that assertion proves the missing behavior, so a compile error is no proof. +func firstRunProblems(readiness string, order map[string]round) []string { const fix = "Fix: run the first scenario at its commit, before the handler, and record that run under ## First failing run " + - "in the lines - **Command**:, - **Exit**:, and - **Assertion**:" - parts, at, found := map[string]string{}, 0, false + "in the lines - **Commit**:, - **Command**:, - **Exit**:, and - **Assertion**:" + parts, at, found, fence := map[string]string{}, 0, false, "" for i, text := range strings.Split(readiness, "\n") { + if trimmed := strings.TrimSpace(text); fence != "" || strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~") { + switch { + case fence == "": + fence = trimmed[:3] + case strings.HasPrefix(trimmed, fence): + fence = "" + } + continue + } if h := readinessHeading.FindStringSubmatch(text); h != nil && len(h[1]) <= 2 { found = found || h[2] == "First failing run" at = 0 @@ -36,6 +47,13 @@ func firstRunProblems(readiness string) []string { return []string{"readiness.md: it holds no part ## First failing run, so no record shows that the first scenario of round 1 failed before the handler. " + fix} } var problems []string + scenario, recorded := order["Scenario"], "" + if len(scenario.cells) == 2 { + recorded, _ = ledgerCommit(scenario.cells[1]) + } + if commit, _ := ledgerCommit(parts["Commit"]); commit == "" || commit != recorded { + problems = append(problems, fmt.Sprintf("readiness.md: ## First failing run names the commit %q, which is not the Scenario commit of ## Order of round 1. %s", parts["Commit"], fix)) + } if parts["Command"] == "" { problems = append(problems, "readiness.md: ## First failing run names no command. "+fix) } diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index f9b62fbd..7fba4bf9 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -28,8 +28,9 @@ var ( // grade, anywhere after the header (#413, #416). The plain words pass, so // a title or prose can say that a call blocks or warns (#419). readinessSeverity = regexp.MustCompile("(?i)`(block|warn|advise)`") - // readinessCell finds a severity alone in a cell of a table (#416). - readinessCell = regexp.MustCompile("(?i)\\|\\s*`?(block|warn|advise)`?\\s*(?:\\||$)") + // readinessGrade finds the word severity before a severity, a grade in + // prose such as "Severity: block", in any part (#420). + readinessGrade = regexp.MustCompile(`(?i)\bseverity\b[^a-z0-9]*\b(block|warn|advise)\b`) // readinessWord finds the word severity under ## Findings (#413). readinessWord = regexp.MustCompile(`(?i)\bseverity\b`) ) @@ -38,18 +39,36 @@ var ( // reviewer sees the word or the table that caused it (#416). func severityCause(text, fence string) string { const fix = "Fix: grade each finding only in its line - **Severity**:, and write " - code, cell := readinessSeverity.FindStringSubmatch(text), readinessCell.FindStringSubmatch(text) + code, cell, grade := readinessSeverity.FindStringSubmatch(text), severityCell(text), readinessGrade.FindStringSubmatch(text) switch { - case cell != nil && strings.HasPrefix(strings.TrimSpace(text), "|"): - return fmt.Sprintf("this table names the severity %s. Fix: remove the table, because each finding gives its severity in its line - **Severity**:", strings.ToLower(cell[1])) + case cell != "": + return fmt.Sprintf("this table names the severity %s. Fix: remove the table, because each finding gives its severity in its line - **Severity**:", cell) case code != nil && fence != "": return fmt.Sprintf("this fenced code names the severity `%s`. %sthe word %s in other text without backticks", strings.ToLower(code[1]), fix, strings.ToLower(code[1])) case code != nil: return fmt.Sprintf("this line names the severity `%s` in code, outside the line - **Severity**: of a finding. %sthe word %s in other text without backticks", strings.ToLower(code[1]), fix, strings.ToLower(code[1])) + case grade != nil: + return fmt.Sprintf("this line grades with the severity %s outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity", strings.ToLower(grade[1])) } return "this line names the word severity outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity" } +// severityCell returns the severity that a cell of a table row holds alone, +// with only marks around it, such as `warn`, **block**, or (advise). It +// returns "" for any other line (#416, #420). +func severityCell(text string) string { + trimmed := strings.TrimSpace(text) + if !strings.HasPrefix(trimmed, "|") { + return "" + } + for _, cell := range strings.Split(trimmed, "|") { + if word := strings.ToLower(strings.Trim(cell, " \t`*_().:!")); severityRank[word] > 0 { + return word + } + } + return "" +} + // readinessFindings reads the findings of readiness.md, outside fenced code. // It reports each finding that finish cannot grade, so a finding in another // form never passes as no finding. Under ## Findings, it also reports each @@ -73,7 +92,7 @@ func readinessFindings(readiness string) ([]finding, []string) { bullet := part > 0 && fence == "" && m != nil && m[1] == "Severity" // The first three lines are the header. A severity elsewhere fails, // also under another part after ## Findings (#416). - if i >= 3 && !bullet && (readinessSeverity.MatchString(text) || readinessCell.MatchString(text) && strings.HasPrefix(trimmed, "|") || + if i >= 3 && !bullet && (readinessSeverity.MatchString(text) || severityCell(text) != "" || readinessGrade.MatchString(text) || part > 0 && !section && readinessWord.MatchString(text)) { problems = append(problems, fmt.Sprintf("readiness.md:%d: %s", i+1, severityCause(text, fence))) } @@ -94,6 +113,8 @@ func readinessFindings(readiness string) ([]finding, []string) { if level == 2 && heading[2] == "Findings" { part, seen = i+1, true } + case level == 3 && part == 0 && readinessID.MatchString(heading[2]): + problems = append(problems, fmt.Sprintf("readiness.md:%d: this finding is outside ## Findings, so finish cannot grade it. %s", i+1, fix)) case level == 3 && part > 0: f := finding{line: i + 1, title: heading[2]} if id := readinessID.FindStringSubmatch(heading[2]); id != nil { diff --git a/sdk/extensionrepo/release.go b/sdk/extensionrepo/release.go index 87f3fa09..385044c2 100644 --- a/sdk/extensionrepo/release.go +++ b/sdk/extensionrepo/release.go @@ -72,15 +72,15 @@ func (r *releaseReceipt) publish(ctx context.Context, env []string, root string, if fixture != "" { acceptance = append(acceptance, fixture) } - if err := r.require(ctx, env, root, "acceptance", acceptance...); err != nil { - return err - } // A receipt that names just finish proves nothing about the repository, // because a hand can write it. So the gate runs again on the signed - // commit, before the tag (#419). + // commit, before acceptance, which can be slow, and before the tag (#419). if err := r.require(ctx, env, root, "finish", "just", "finish"); err != nil { return err } + if err := r.require(ctx, env, root, "acceptance", acceptance...); err != nil { + return err + } // Acceptance and finish cannot change the source that the receipt signed. if err := releasePushed(ctx, env, root, r.SHA); err != nil { return err From 1289fc1f6c93befdf44d32a286273ff3caf49678 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 20:12:05 -0400 Subject: [PATCH 32/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20template:=20the=20?= =?UTF-8?q?record=20of=20the=20first=20failing=20run,=20the=20forms=20that?= =?UTF-8?q?=20grade,=20and=20the=20order=20of=20release?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docs follow the fix before. finish-gate.md keeps its list of failures in one piece, and names the Commit line of ## First failing run and the forms that grade. readiness-review.md lists those forms, says what to do after just update, and keeps a sentence out of the last calibration example. The rule index gets the three rules it missed and links ledger_run.go. Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/extensions.md | 4 ++-- docs/north-star/contract/finish-gate.md | 12 +++++++----- docs/north-star/contract/test-kit.md | 2 +- .../skills/nicely-extension/readiness-review.md | 12 +++++++++++- template/AGENTS.md.jinja | 2 +- template/AGENTS/rule-index.md.jinja | 7 +++++-- 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/docs/north-star/contract/extensions.md b/docs/north-star/contract/extensions.md index fe583319..c60291f5 100644 --- a/docs/north-star/contract/extensions.md +++ b/docs/north-star/contract/extensions.md @@ -24,8 +24,8 @@ An author releases a version with `just release vX.Y.Z [fixture]`. The SDK requires a clean HEAD equal to the fetched default branch of `origin`. A successful signoff receipt must name that SHA, with exit 0 for every check and signoff. Its checks must hold `just finish`, so a receipt with only `just check` never releases. -Release runs `just acceptance [fixture]`, then `just finish` again on that clean HEAD, because a hand can write a receipt (#419). -Then it creates and pushes an annotated tag whose message holds that signoff receipt. +Release runs `just finish` again on that clean HEAD, because a hand can write a receipt (#419). +Then it runs `just acceptance [fixture]`, and creates and pushes an annotated tag whose message holds that signoff receipt. It executes the fenced `sh` block under `## Install` in `README.md`, with `TAG` replaced by the new tag. The install uses scratch HOME and XDG folders, without the caller's keys or installed extensions. The clone's HEAD must equal the tagged SHA, even when another README paragraph names the new tag. diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index 99879ff4..59086a2e 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -12,12 +12,13 @@ This part says what `just finish` checks in an extension repository, as [Extensi - A `readiness.md` whose first three lines hold no `Commit: `, no `Reviewer: `, or no verdict `Ready` or `Ready with stated limits`. A byte order mark before them is ignored - A reviewed commit that differs from the working tree outside `readiness.md`, and outside the part **Limits.** of `spec.md`. So the user accepts a limit after a review, with no new round - A finding of `readiness.md` without its own `###` heading under `## Findings`, or without one severity: `block`, `warn`, or `advise` -- A severity after the first three lines, outside the line `- **Severity**:` of a finding, in a form that grades. Such a form is code, such as `` `warn` ``, also in fenced code, or a cell of a table that holds only a severity. Under `## Findings`, the word severity fails too. The message names the word or the table that caused it +- A severity after the first three lines, outside the line `- **Severity**:` of a finding, in a form that grades. Such a form is code, such as `` `warn` ``, also in fenced code, or a cell of a table that holds only a severity, with marks such as `**` around it. The word severity before a severity, such as "Severity: block", is a form that grades too. Under `## Findings`, the word severity fails alone. The message names the word or the table that caused it +- A finding heading with an ID, such as `### R1-F2`, outside `## Findings` +- `Ready` with a `block` or `warn` finding that is not `fixed`, or with a limit in `spec.md` +- `Ready with stated limits` with a `block` finding that is not `fixed`, or with a `warn` that is neither `fixed` nor an accepted `limit` Finish reads a severity only in the line `- **Severity**:` of a finding. The plain words block, warn, and advise pass in a title, in evidence, and in prose, such as `### R1-F1 Large inputs block the terminal` (#419). -- `Ready` with a `block` or `warn` finding that is not `fixed`, or with a limit in `spec.md` -- `Ready with stated limits` with a `block` finding that is not `fixed`, or with a `warn` that is neither `fixed` nor an accepted `limit` ## The ledger @@ -37,8 +38,9 @@ A round before the last whose Finish is `pass` ends a build. The next round star Under `## Order of round 1`, a table with the columns Step and Commit names the commit of each step of the current build: Intake, Spec, Scenario, and Handler. -Under `## First failing run`, three lines record the run of the first scenario of the current build at its commit, before the handler (#420): +Under `## First failing run`, four lines record the run of the first scenario of the current build at its commit, before the handler (#420): +- `- **Commit**:` the Scenario commit of `## Order of round 1`, so a record of an earlier build never stands in - `- **Command**:` the command that ran the scenario - `- **Exit**:` its exit status, which is not 0 - `- **Assertion**:` the assertion of the scenario that failed @@ -68,7 +70,7 @@ A finding of the last round can have no state yet. - A commit of a round or of a fixed finding that is not HEAD or an ancestor of HEAD - A last round whose Head is not the `Commit:` of `readiness.md` - A step of round 1 that is missing, out of order, or in the commit of the step before it -- No part `## First failing run`, or one without its command, without an exit status other than 0, or without its assertion +- No part `## First failing run` outside fenced code. Also a part without the Scenario commit, without its command, without an exit status other than 0, or without its assertion - A finding without an ID, a finding of an earlier round without a state, or a state without its parts - A finding with the state `open`, even when `## Open lines` holds `None.` - A finding with the state `open` whose ID no line under `## Open lines` of `spec.md` names. So `spec.md` and the ledger agree on the open work (#420) diff --git a/docs/north-star/contract/test-kit.md b/docs/north-star/contract/test-kit.md index 0449ca3d..f427043e 100644 --- a/docs/north-star/contract/test-kit.md +++ b/docs/north-star/contract/test-kit.md @@ -56,7 +56,7 @@ Each case of the Output group of [Generic cases](generic-cases.md) that applies `acceptance.sh` makes that file in the extension's own domain, and proves the safe outcome on it. Two cases need more: - **Keeps data that a reader needs.** The test file holds a valid block that a reader needs, and that block holds private text. The check proves both results: the private text is not in the output, and the output still shows the content correctly -- **Loses data that changes the meaning.** The test file gives each declaration that changes how a reader shows the content a value that is not the default. The independent reference applies that declaration before it compares the content, so a lost declaration fails the check +- **Loses data that changes the meaning.** Some declarations change how a reader shows the content. The test file gives each of them a value that is not the default. The independent reference applies that declaration before it compares the content, so a lost declaration fails the check The tests make each file that they use. They never copy a file of the user, and the author never searches the home folder of the user for sample files. diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 392901ba..dc405e8f 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -37,7 +37,14 @@ Write each part of a finding as a bullet that starts with its bold name, such as - **Fix**: the smallest change that removes the problem Name a severity only in the bullet **Severity** of its finding. `just finish` reads no other line as a grade. -After the first three lines, it fails on a severity in code, such as `` `warn` ``, on a severity alone in a cell of a table, and on the word severity under `## Findings`. +After the first three lines, it fails on each form that grades in another place: + +- A severity in code, such as `` `warn` `` +- A cell of a table that holds only a severity +- The word severity before a severity, such as "Severity: block" +- A finding heading, such as `### R1-F2`, outside `## Findings` +- The word severity under `## Findings` + The plain words pass in a title, in evidence, and in prose, such as `### R1-F1 Large inputs block the terminal`. A loss that a user can see is `warn` or `block`, never `advise`. @@ -49,6 +56,7 @@ Grade it so in the round that first reports it. These examples set the grade: - **A message that names a wrong cause.** `inspect` answers that `notes.md` is missing, but the file exists and is not readable. At least `warn`, because the agent fixes the wrong thing - **A dry run that predicts a result that the run does not give.** The dry run says that the output is `notes.txt`, and the run writes `notes.md.txt`. At least `warn`. `block` when the run replaces a file that the dry run said it keeps - **An output whose meaning changes.** `notes.txt` drops the line `encoding: latin-1` and keeps the bytes, so a reader shows other letters. At least `warn`. `block` when `spec.md` promises the same text + Never lower the severity of a finding in a later round. A `warn` or `block` finding closes only as `fixed` or `limit`. Only an `advise` can be `declined`. @@ -101,6 +109,7 @@ After the findings, keep the ledger of the build. Add one row for this round, an ## First failing run +- **Commit**: - **Command**: - **Exit**: - **Assertion**: @@ -115,5 +124,6 @@ A build has at most 5 rounds. A round whose Finish is `pass` ends a build. A new build, such as of another command, starts at a new intake after it. The round numbers go on, and the ledger keeps each finding of each build. After `just update` from a template without a ledger, keep each finding of the older reviews, with an ID and a state. +After `just update` from a template without `## First failing run`, run the scenario of the current build again at its commit. Then record that run. Never reject a script because it is small, young, or private. Judge what the command does, not where the script comes from. diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index a09a0450..a71d141e 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -103,7 +103,7 @@ It never replaces "Done when". `just signoff` runs `just finish` first, so a mer 3. Run `just signoff` for that pushed commit 4. Make sure that "Done when" holds for this commit 5. Merge that exact signed SHA into the default branch -6. Run `just release vX.Y.Z`, with an optional fixture path. It runs `just acceptance`, then `just finish` again on that commit, before it makes the tag +6. Run `just release vX.Y.Z`, with an optional fixture path. It runs `just finish` again on that commit, then `just acceptance`, before it makes the tag 7. Read the printed receipt for acceptance, the published install, Economy reservations, hosted checks, and uninstall For release recovery, read "Release and install" in the pinned contract's `extensions.md`. diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index 0c234484..3d116f8b 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -22,8 +22,9 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | One bounded job, no arbitrary argument relay; the skill teaches when | Prose only | | Round 1 records the intake, then the spec, then a failing scenario, then the handler | [Finish gate][finish] checks their commits in the [ledger][ledger] | | The intake gives, for each fact, its source, its version, and the part read, with each experiment apart; a link alone is not evidence | Prose only | -| Round 1 records the command, the exit status, and the failed assertion of the first scenario run | The [ledger][ledger] checks the record | -| A compile error is no failure of the first scenario | Prose only | +| Round 1 records the Scenario commit, the command, the exit status, and the failed assertion of the first scenario run | The [ledger][ledger-run] checks the record | +| A compile error is no failure of the first scenario; a scenario that cannot run without the command commits a handler that does no work | Prose only | +| Read the source first, then run experiments only to confirm what you read | Prose only | | Each case of the Output group that applies has a test file that the tests make, checked by an independent reader | Prose only | | The tests make their files; never search the home folder or copy a file of the user | Prose only | | Round 1 tests each generic case that applies, and the first review tests families of cases | Prose only | @@ -40,6 +41,7 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | Each finding keeps its ID and its highest severity, and gets a state each round; a `warn` or `block` closes only as `fixed` or `limit` | [Ledger findings][ledger-findings], [ledger history][ledger-history] | | `## Open lines` of `spec.md` names each finding that the ledger records as `open`, by its ID | [Ledger findings][ledger-findings] | | A severity is read only in the line **Severity** of a finding; the plain words block, warn, and advise pass elsewhere | [Readiness findings][readiness] | +| A finding heading stays under `## Findings`, and no other form grades | [Readiness findings][readiness] | | A broken promise of `spec.md` or of the help text is at least `warn` at its first review | Prose only | | Each round keeps its row and has a committed review; a build ends only at a review that passes | [Ledger history][ledger-history] | | Accepting a limit starts no round; after a review, only **Limits.** of `spec.md` changes | [Finish gate][finish-verdict] | @@ -58,6 +60,7 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". [finish]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/finish.go [ledger]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger.go +[ledger-run]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger_run.go [ledger-findings]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger_findings.go [ledger-history]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger_history.go [finish-verdict]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/readiness_verdict.go From 4f425210874c6f0f05bedd1e28b6674811b65201 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 20:31:24 -0400 Subject: [PATCH 33/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20no=20fi?= =?UTF-8?q?nding=20ID=20hides=20outside=20##=20Findings,=20and=20the=20run?= =?UTF-8?q?=20record=20reads=20like=20a=20finding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A second independent review found these gaps: - A line outside ## Findings, such as "- R3-F8 Long lines are cut: block", hid a graded finding. Finish now fails on each finding ID outside ## Findings that no finding under it has - The record of the first failing run closed a fence of four marks at a line of three, and refused a value that ends with a period. It now reads fences and values as readinessFindings does - The proof of a forged receipt now checks that release ran no acceptance before just finish refused, and the proof of the hidden forms checks each message again Co-Authored-By: Claude Opus 5.5 --- scripts/template-finish-test.sh | 5 +++-- scripts/template-ledger-builds-test.sh | 3 ++- scripts/template-ledger-run-test.sh | 3 +++ scripts/template-release-test.sh | 3 +++ sdk/extensionrepo/ledger_run.go | 14 ++++++++------ sdk/extensionrepo/readiness.go | 19 +++++++++++++++++++ 6 files changed, 38 insertions(+), 9 deletions(-) diff --git a/scripts/template-finish-test.sh b/scripts/template-finish-test.sh index def93617..6895b311 100644 --- a/scripts/template-finish-test.sh +++ b/scripts/template-finish-test.sh @@ -123,9 +123,10 @@ fct_test_finish() { fct_finish_said 'readiness.md:13: this table names the severity warn' fct_finish_said 'readiness.md:14: this table names the severity block' fct_finish_said 'readiness.md:19: this fenced code names the severity `warn`' - for number in 24 29 33 36; do - fct_finish_said "readiness.md:${number}: this line " + for number in 24 29 33; do + fct_finish_said "readiness.md:${number}: this line grades with the severity warn" done + fct_finish_said 'readiness.md:36: this line names the severity `block` in code' # Finish reads Commit:, Reviewer:, and Verdict: only in the first lines, never in fenced code (#413). fct_readiness "${sha}" 'Ready with stated limits' $'```\nReviewer: an agent that wrote none of the code\n```\n\nNone.' '' diff --git a/scripts/template-ledger-builds-test.sh b/scripts/template-ledger-builds-test.sh index d2e69295..834b0e3b 100644 --- a/scripts/template-ledger-builds-test.sh +++ b/scripts/template-ledger-builds-test.sh @@ -45,10 +45,11 @@ fct_test_ledger_builds() { fct_finish_said 'this line names the severity `warn` in code, outside the line - **Severity**: of a finding' fct_finish_said 'this table names the severity warn' # A grade in prose, a finding outside ## Findings, or a severity with marks in a cell fails in any part (#420). - fct_readiness "${head}" Ready "${valid}"$'\n\n## Notes\n\n- Long lines are cut. Severity: block.\n\n## Other issues\n\n### R3-F9 Long lines are cut\n\n| R3-F9 | **Block.** |' 'agent B' + fct_readiness "${head}" Ready "${valid}"$'\n\n## Notes\n\n- Long lines are cut. Severity: block.\n- R3-F8 Long lines are cut: block\n\n## Other issues\n\n### R3-F9 Long lines are cut\n\n| R3-F9 | **Block.** |' 'agent B' fct_finish 1 fct_finish_said 'this line grades with the severity block outside the line - **Severity**: of a finding' fct_finish_said 'this finding is outside ## Findings, so finish cannot grade it' + fct_finish_said 'this line names the finding R3-F8, which has no ### heading under ## Findings' fct_finish_said 'this table names the severity block' # Finish reads a severity only in the line - **Severity**:, so the words # block and warn pass in a title, in evidence, and in prose (#419). diff --git a/scripts/template-ledger-run-test.sh b/scripts/template-ledger-run-test.sh index a8493c3a..fe9fa854 100644 --- a/scripts/template-ledger-run-test.sh +++ b/scripts/template-ledger-run-test.sh @@ -37,5 +37,8 @@ fct_test_ledger_run() { ledger_run=$'```\n- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `go test ./...`\n- **Exit**: 1\n- **Assertion**: `stdout 3`\n```' fct_readiness "${head}" Ready "${valid}" 'agent B' fct_finish 1 fct_finish_said 'names no command' + # A longer fence holds a shorter one, and a value can end with a period. + ledger_run=$'````\n```\n````\n\n- **Commit**: '"$(fct_step "${finish}" scenario)"$'.\n- **Command:** `go test ./...`\n- **Exit**: 1.\n- **Assertion**: `stdout 3` found no match.' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 0 git -C "${finish}" checkout -q readiness.md } diff --git a/scripts/template-release-test.sh b/scripts/template-release-test.sh index abe09309..ee3f4be8 100644 --- a/scripts/template-release-test.sh +++ b/scripts/template-release-test.sh @@ -57,6 +57,9 @@ SCRIPT fct_fail 'release trusts a receipt that names just finish, on a repository that just finish refuses' fi [[ -z "$(git -C "${remote}" tag -l)" ]] || fct_fail 'a release that just finish refuses pushed a tag' + receipt="$(sed -n 's/^release: receipt in //p' "${scratch}/release.out")" + PATH="/usr/bin:${PATH}" jq -e '[.steps[].name] | index("finish") != null and index("acceptance") == null' "${receipt}" >/dev/null || + fct_fail 'release ran acceptance before just finish refused the repository' # The releases below pass just finish: the forms are filled, and an independent review passes. fct_steps "${repo}" diff --git a/sdk/extensionrepo/ledger_run.go b/sdk/extensionrepo/ledger_run.go index cb9a9de9..700b872f 100644 --- a/sdk/extensionrepo/ledger_run.go +++ b/sdk/extensionrepo/ledger_run.go @@ -22,14 +22,16 @@ func firstRunProblems(readiness string, order map[string]round) []string { "in the lines - **Commit**:, - **Command**:, - **Exit**:, and - **Assertion**:" parts, at, found, fence := map[string]string{}, 0, false, "" for i, text := range strings.Split(readiness, "\n") { - if trimmed := strings.TrimSpace(text); fence != "" || strings.HasPrefix(trimmed, "```") || strings.HasPrefix(trimmed, "~~~") { - switch { - case fence == "": - fence = trimmed[:3] - case strings.HasPrefix(trimmed, fence): + // A fence closes with its own run of marks, alone on its line, as in readinessFindings. + switch trimmed := strings.TrimSpace(text); { + case fence != "": + if strings.HasPrefix(trimmed, fence) && strings.Trim(trimmed, fence[:1]) == "" { fence = "" } continue + case strings.HasPrefix(trimmed, "```"), strings.HasPrefix(trimmed, "~~~"): + fence = trimmed[:len(trimmed)-len(strings.TrimLeft(trimmed, trimmed[:1]))] + continue } if h := readinessHeading.FindStringSubmatch(text); h != nil && len(h[1]) <= 2 { found = found || h[2] == "First failing run" @@ -40,7 +42,7 @@ func firstRunProblems(readiness string, order map[string]round) []string { continue } if m := firstRun.FindStringSubmatch(text); at > 0 && m != nil { - parts[m[1]] = strings.Trim(m[2], "`") + parts[m[1]] = strings.Trim(strings.TrimSuffix(m[2], "."), "`") } } if !found { diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index 7fba4bf9..fd10d1c4 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -3,6 +3,7 @@ package extensionrepo import ( "cmp" "fmt" + "maps" "regexp" "slices" "strings" @@ -24,6 +25,8 @@ var ( readinessHeading = regexp.MustCompile(`^(#{1,6})\s+(.+?)\s*$`) readinessPart = regexp.MustCompile(`^\s*[-*+]\s+\*\*(Severity|State|Spec|Commit|Reason|Limit)(?:\*\*:|:\*\*)\s*(.*?)\s*$`) readinessID = regexp.MustCompile(`^(R([1-9][0-9]*)-F[1-9][0-9]*)\b`) + // readinessMention finds the ID of a finding anywhere in a line. + readinessMention = regexp.MustCompile(`\bR[1-9][0-9]*-F[1-9][0-9]*\b`) // readinessSeverity finds a severity in code, a form that finish cannot // grade, anywhere after the header (#413, #416). The plain words pass, so // a title or prose can say that a call blocks or warns (#419). @@ -78,6 +81,8 @@ func readinessFindings(readiness string) ([]finding, []string) { var findings []finding var problems []string part, current, fence, opened, none, stray, seen := 0, -1, "", 0, false, false, false + // mentions holds the line of each finding ID outside ## Findings. + mentions := map[string]int{} closeFindings := func() { if part > 0 && len(findings) == 0 && !none { problems = append(problems, fmt.Sprintf("readiness.md:%d: ## Findings holds no ### finding and no None. %s", part, fix)) @@ -96,6 +101,13 @@ func readinessFindings(readiness string) ([]finding, []string) { part > 0 && !section && readinessWord.MatchString(text)) { problems = append(problems, fmt.Sprintf("readiness.md:%d: %s", i+1, severityCause(text, fence))) } + if i >= 3 && part == 0 && heading == nil { + for _, id := range readinessMention.FindAllString(text, -1) { + if _, found := mentions[id]; !found { + mentions[id] = i + 1 + } + } + } switch { case fence != "": if strings.HasPrefix(trimmed, fence) && strings.Trim(trimmed, fence[:1]) == "" { @@ -173,6 +185,13 @@ func readinessFindings(readiness string) ([]finding, []string) { if f.severity == "" { problems = append(problems, fmt.Sprintf("readiness.md:%d: this finding has no severity. %s", f.line, fix)) } + delete(mentions, f.id) + } + // An ID outside ## Findings names a finding that finish grades, so no + // finding hides in another part (#420). + ids := slices.SortedFunc(maps.Keys(mentions), func(a, b string) int { return mentions[a] - mentions[b] }) + for _, id := range ids { + problems = append(problems, fmt.Sprintf("readiness.md:%d: this line names the finding %s, which has no ### heading under ## Findings. %s", mentions[id], id, fix)) } return findings, problems } From 1fa2f5146b4016c4f2ec28d99ccb28279f29fc6c Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 20:31:26 -0400 Subject: [PATCH 34/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20template:=20finish?= =?UTF-8?q?-gate=20names=20stray=20finding=20IDs,=20and=20verify-nicely=20?= =?UTF-8?q?reads=20intake.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit finish-gate.md and readiness-review.md name the check of a finding ID outside ## Findings, and finish-gate.md splits a sentence of 30 words. The step "Loop text" of verify-nicely reads the parts of the skill after the move of the intake to intake.md. intake.md limits experiments to commands that neither write nor bill. Co-Authored-By: Claude Opus 5.5 --- .agents/skills/verify-nicely/features/template.md | 2 +- docs/north-star/contract/finish-gate.md | 4 ++-- template/.agents/skills/nicely-extension/intake.md | 2 +- template/.agents/skills/nicely-extension/readiness-review.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.agents/skills/verify-nicely/features/template.md b/.agents/skills/verify-nicely/features/template.md index 54b92ab3..badced82 100644 --- a/.agents/skills/verify-nicely/features/template.md +++ b/.agents/skills/verify-nicely/features/template.md @@ -42,7 +42,7 @@ Preconditions: - **Remove command.** In the output of `skill view nicely-extensions`, the skill holds the section `## Remove an extension`. It names `ncly doctor extensions --json`, the four files `ncly-SLUG`, `ncly-SLUG.toml`, `ncly-SLUG.locales/`, and `ncly-SLUG.skill/`, `just uninstall`, `NOT_FOUND`, and `GRANT_UNUSED`. - **Render and install.** Run `just verify prepare "$run" template`. It exits 0. Its log names `just install` of `probe`, the line `installed /scratch/home/.local/share/nicely/extensions/ncly-probe`, and the `python3` that it puts first in the PATH of the wrapper. Then run `ls "$run/scratch/home/.local/share/nicely/extensions"`: it lists `ncly-probe`, `ncly-probe.locales`, `ncly-probe.skill`, and `ncly-probe.toml`, beside the four files of `hello`. Report the output of `prepare` and of `ls` beside the quote. - **Release and install text.** Run `grep -n -A12 '^## Release$' "$run/scratch/ncly-probe/AGENTS.md"` and `grep -n -A3 '^git clone' "$run/scratch/ncly-probe/README.md"`. Both exit 0. `AGENTS.md` names `just release vX.Y.Z` and the release receipt. `README.md` holds the lines `git clone --branch TAG https://example.com/ncly-probe ncly-probe`, `cd ncly-probe`, `just install`, and `ncly describe probe --json`. Report both outputs beside the quote. -- **Loop text.** Run `grep -n -A10 '^## Done when$' "$run/scratch/ncly-probe/AGENTS.md"`, then `grep -n '^## Open lines$' "$run/scratch/ncly-probe/spec.md"`. Both exit 0. "Done when" names `just finish`, the hosted checks, 5 rounds, and a "full stop" request that runs only after it. Then run `grep -n '^##' "$run/scratch/ncly-probe/.agents/skills/nicely-extension/SKILL.md"`. It lists `## Intake of a program`, `## Acceptance lines`, `## Rounds`, and `## Readiness review`. Report the three outputs beside the quote. +- **Loop text.** Run `grep -n -A10 '^## Done when$' "$run/scratch/ncly-probe/AGENTS.md"`, then `grep -n '^## Open lines$' "$run/scratch/ncly-probe/spec.md"`. Both exit 0. "Done when" names `just finish`, the hosted checks, 5 rounds, and a "full stop" request that runs only after it. Then run `grep -n '^##' "$run/scratch/ncly-probe/.agents/skills/nicely-extension/SKILL.md"`. It lists `## Intake`, `## Acceptance lines`, `## Test files`, `## Rounds`, and `## Readiness review`. Then run `grep -n '^##' "$run/scratch/ncly-probe/.agents/skills/nicely-extension/intake.md"`. It lists `## Intake of a script`, `## Intake of a program`, and `## Evidence of the intake`. Report the four outputs beside the quote. - **Finish gate.** Run `(cd "$run/scratch/ncly-probe" && just finish)`, whose subshell keeps the folder of the caller. Exit 1. Stderr names each `Fill:` line of `vision.md`, `spec.md`, and `qa.md` as `:: this line of the form is not filled`, and `readiness.md: no readiness verdict is recorded`. Report the output beside the quote. - **Discovery.** Run `just verify step "$run" template --help`. Exit 0, and the `EXTENSIONS` section lists `probe: Inspect files` beside `hello` and `skill`. - **Help.** Run `just verify step "$run" template probe --help`. Exit 0, and `COMMANDS` lists `inspect: Give the size and the line count of a file`. diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index 59086a2e..bcaddea1 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -12,8 +12,8 @@ This part says what `just finish` checks in an extension repository, as [Extensi - A `readiness.md` whose first three lines hold no `Commit: `, no `Reviewer: `, or no verdict `Ready` or `Ready with stated limits`. A byte order mark before them is ignored - A reviewed commit that differs from the working tree outside `readiness.md`, and outside the part **Limits.** of `spec.md`. So the user accepts a limit after a review, with no new round - A finding of `readiness.md` without its own `###` heading under `## Findings`, or without one severity: `block`, `warn`, or `advise` -- A severity after the first three lines, outside the line `- **Severity**:` of a finding, in a form that grades. Such a form is code, such as `` `warn` ``, also in fenced code, or a cell of a table that holds only a severity, with marks such as `**` around it. The word severity before a severity, such as "Severity: block", is a form that grades too. Under `## Findings`, the word severity fails alone. The message names the word or the table that caused it -- A finding heading with an ID, such as `### R1-F2`, outside `## Findings` +- A severity after the first three lines, outside the line `- **Severity**:` of a finding, in a form that grades. Such a form is code, such as `` `warn` ``, also in fenced code. A cell of a table that holds only a severity is such a form too, also with marks such as `**` around it. The word severity before a severity, such as "Severity: block", is a form that grades too. Under `## Findings`, the word severity fails alone. The message names the word or the table that caused it +- A finding heading with an ID, such as `### R1-F2`, outside `## Findings`. Also a line outside `## Findings` that names the ID of no finding under it - `Ready` with a `block` or `warn` finding that is not `fixed`, or with a limit in `spec.md` - `Ready with stated limits` with a `block` finding that is not `fixed`, or with a `warn` that is neither `fixed` nor an accepted `limit` diff --git a/template/.agents/skills/nicely-extension/intake.md b/template/.agents/skills/nicely-extension/intake.md index a175c632..0b88bde5 100644 --- a/template/.agents/skills/nicely-extension/intake.md +++ b/template/.agents/skills/nicely-extension/intake.md @@ -33,4 +33,4 @@ The intake records what you read, not only where the facts are (#420): - Each declaration of the input that changes how a reader shows the content, not only the file types. `generic-cases.md` in the contract says why - Each fact that no source gives, as an open line of `spec.md` -A link alone is not evidence of a read. Read the source first. Then run experiments only to confirm what you read. +A link alone is not evidence of a read. Read the source first. Then run experiments only to confirm what you read, with a command that neither writes nor bills. diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index dc405e8f..9480c63b 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -42,7 +42,7 @@ After the first three lines, it fails on each form that grades in another place: - A severity in code, such as `` `warn` `` - A cell of a table that holds only a severity - The word severity before a severity, such as "Severity: block" -- A finding heading, such as `### R1-F2`, outside `## Findings` +- A finding heading, such as `### R1-F2`, outside `## Findings`, or an ID outside it of no finding under it - The word severity under `## Findings` The plain words pass in a title, in evidence, and in prose, such as `### R1-F1 Large inputs block the terminal`. From 26191363c4d2ac33b1c498f14936c31833e631e1 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 20:38:38 -0400 Subject: [PATCH 35/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20a=20hea?= =?UTF-8?q?ding=20outside=20##=20Findings=20names=20no=20stray=20finding?= =?UTF-8?q?=20ID=20either?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The third independent review found that a heading, such as "## R3-F7 Long lines are cut" or "#### R3-F6", still carried a finding ID outside ## Findings with no grade. Such a heading now counts as a mention. A heading of level 3 with an ID keeps its own message. A test proves that an honest summary of existing IDs still passes. The step "Loop text" of verify-nicely reads 12 lines after "## Done when", so it reaches the line of a "full stop" request, and a sentence of finish-gate.md is split. Co-Authored-By: Claude Opus 5.5 --- .agents/skills/verify-nicely/features/template.md | 2 +- docs/north-star/contract/finish-gate.md | 3 ++- scripts/template-ledger-builds-test.sh | 6 ++++-- sdk/extensionrepo/readiness.go | 5 ++++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.agents/skills/verify-nicely/features/template.md b/.agents/skills/verify-nicely/features/template.md index badced82..774f346f 100644 --- a/.agents/skills/verify-nicely/features/template.md +++ b/.agents/skills/verify-nicely/features/template.md @@ -42,7 +42,7 @@ Preconditions: - **Remove command.** In the output of `skill view nicely-extensions`, the skill holds the section `## Remove an extension`. It names `ncly doctor extensions --json`, the four files `ncly-SLUG`, `ncly-SLUG.toml`, `ncly-SLUG.locales/`, and `ncly-SLUG.skill/`, `just uninstall`, `NOT_FOUND`, and `GRANT_UNUSED`. - **Render and install.** Run `just verify prepare "$run" template`. It exits 0. Its log names `just install` of `probe`, the line `installed /scratch/home/.local/share/nicely/extensions/ncly-probe`, and the `python3` that it puts first in the PATH of the wrapper. Then run `ls "$run/scratch/home/.local/share/nicely/extensions"`: it lists `ncly-probe`, `ncly-probe.locales`, `ncly-probe.skill`, and `ncly-probe.toml`, beside the four files of `hello`. Report the output of `prepare` and of `ls` beside the quote. - **Release and install text.** Run `grep -n -A12 '^## Release$' "$run/scratch/ncly-probe/AGENTS.md"` and `grep -n -A3 '^git clone' "$run/scratch/ncly-probe/README.md"`. Both exit 0. `AGENTS.md` names `just release vX.Y.Z` and the release receipt. `README.md` holds the lines `git clone --branch TAG https://example.com/ncly-probe ncly-probe`, `cd ncly-probe`, `just install`, and `ncly describe probe --json`. Report both outputs beside the quote. -- **Loop text.** Run `grep -n -A10 '^## Done when$' "$run/scratch/ncly-probe/AGENTS.md"`, then `grep -n '^## Open lines$' "$run/scratch/ncly-probe/spec.md"`. Both exit 0. "Done when" names `just finish`, the hosted checks, 5 rounds, and a "full stop" request that runs only after it. Then run `grep -n '^##' "$run/scratch/ncly-probe/.agents/skills/nicely-extension/SKILL.md"`. It lists `## Intake`, `## Acceptance lines`, `## Test files`, `## Rounds`, and `## Readiness review`. Then run `grep -n '^##' "$run/scratch/ncly-probe/.agents/skills/nicely-extension/intake.md"`. It lists `## Intake of a script`, `## Intake of a program`, and `## Evidence of the intake`. Report the four outputs beside the quote. +- **Loop text.** Run `grep -n -A12 '^## Done when$' "$run/scratch/ncly-probe/AGENTS.md"`, then `grep -n '^## Open lines$' "$run/scratch/ncly-probe/spec.md"`. Both exit 0. "Done when" names `just finish`, the hosted checks, 5 rounds, and a "full stop" request that runs only after it. Then run `grep -n '^##' "$run/scratch/ncly-probe/.agents/skills/nicely-extension/SKILL.md"`. It lists `## Intake`, `## Acceptance lines`, `## Test files`, `## Rounds`, and `## Readiness review`. Then run `grep -n '^##' "$run/scratch/ncly-probe/.agents/skills/nicely-extension/intake.md"`. It lists `## Intake of a script`, `## Intake of a program`, and `## Evidence of the intake`. Report the four outputs beside the quote. - **Finish gate.** Run `(cd "$run/scratch/ncly-probe" && just finish)`, whose subshell keeps the folder of the caller. Exit 1. Stderr names each `Fill:` line of `vision.md`, `spec.md`, and `qa.md` as `:: this line of the form is not filled`, and `readiness.md: no readiness verdict is recorded`. Report the output beside the quote. - **Discovery.** Run `just verify step "$run" template --help`. Exit 0, and the `EXTENSIONS` section lists `probe: Inspect files` beside `hello` and `skill`. - **Help.** Run `just verify step "$run" template probe --help`. Exit 0, and `COMMANDS` lists `inspect: Give the size and the line count of a file`. diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index bcaddea1..790f8891 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -18,7 +18,8 @@ This part says what `just finish` checks in an extension repository, as [Extensi - `Ready with stated limits` with a `block` finding that is not `fixed`, or with a `warn` that is neither `fixed` nor an accepted `limit` Finish reads a severity only in the line `- **Severity**:` of a finding. -The plain words block, warn, and advise pass in a title, in evidence, and in prose, such as `### R1-F1 Large inputs block the terminal` (#419). +The plain words block, warn, and advise pass in a title, in evidence, and in prose (#419). +An example is the title `### R1-F1 Large inputs block the terminal`. ## The ledger diff --git a/scripts/template-ledger-builds-test.sh b/scripts/template-ledger-builds-test.sh index 834b0e3b..8d60b0a9 100644 --- a/scripts/template-ledger-builds-test.sh +++ b/scripts/template-ledger-builds-test.sh @@ -45,15 +45,17 @@ fct_test_ledger_builds() { fct_finish_said 'this line names the severity `warn` in code, outside the line - **Severity**: of a finding' fct_finish_said 'this table names the severity warn' # A grade in prose, a finding outside ## Findings, or a severity with marks in a cell fails in any part (#420). - fct_readiness "${head}" Ready "${valid}"$'\n\n## Notes\n\n- Long lines are cut. Severity: block.\n- R3-F8 Long lines are cut: block\n\n## Other issues\n\n### R3-F9 Long lines are cut\n\n| R3-F9 | **Block.** |' 'agent B' + fct_readiness "${head}" Ready "${valid}"$'\n\n## Notes\n\n- Long lines are cut. Severity: block.\n- R3-F8 Long lines are cut: block\n\n## R3-F7 Long lines are cut\n\n## Other issues\n\n### R3-F9 Long lines are cut\n\n| R3-F9 | **Block.** |\n\n#### R3-F6 Long lines are cut' 'agent B' fct_finish 1 fct_finish_said 'this line grades with the severity block outside the line - **Severity**: of a finding' fct_finish_said 'this finding is outside ## Findings, so finish cannot grade it' fct_finish_said 'this line names the finding R3-F8, which has no ### heading under ## Findings' + fct_finish_said 'this line names the finding R3-F7, which has no ### heading under ## Findings' + fct_finish_said 'this line names the finding R3-F6, which has no ### heading under ## Findings' fct_finish_said 'this table names the severity block' # Finish reads a severity only in the line - **Severity**:, so the words # block and warn pass in a title, in evidence, and in prose (#419). - fct_readiness "${head}" Ready "${valid}"$'\n- **Evidence**: the code block in README.md\n\nThe CLI does not warn when the input is empty.\n\n### R3-F1 Large inputs block the terminal\n\n- **Severity**: `advise`\n\n## Notes\n\nA long line can block a pager.' 'agent B' + fct_readiness "${head}" Ready "${valid}"$'\n- **Evidence**: the code block in README.md\n\nThe CLI does not warn when the input is empty.\n\n### R3-F1 Large inputs block the terminal\n\n- **Severity**: `advise`\n\n## Notes\n\nA long line can block a pager. R3-F1 and R2-F1 name such a case.' 'agent B' fct_finish 0 # Finish reads the header in the first three lines, after a byte order mark (#416). fct_readiness "${head}" Ready "${valid}" 'agent B' diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index fd10d1c4..ef10f4d5 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -101,7 +101,10 @@ func readinessFindings(readiness string) ([]finding, []string) { part > 0 && !section && readinessWord.MatchString(text)) { problems = append(problems, fmt.Sprintf("readiness.md:%d: %s", i+1, severityCause(text, fence))) } - if i >= 3 && part == 0 && heading == nil { + // A heading of level 3 with an ID outside ## Findings has its own + // message below, so only other lines count as a mention. + idHeading := heading != nil && len(heading[1]) == 3 && readinessID.MatchString(heading[2]) + if i >= 3 && (part == 0 || section) && !idHeading { for _, id := range readinessMention.FindAllString(text, -1) { if _, found := mentions[id]; !found { mentions[id] = i + 1 From 2f813aa8fb91df9094ebd786a8d510269dea882e Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sat, 31 Oct 2026 20:44:20 -0400 Subject: [PATCH 36/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20a=20fin?= =?UTF-8?q?ding=20heading=20in=20fenced=20code=20outside=20##=20Findings?= =?UTF-8?q?=20is=20a=20stray=20ID=20too?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review of the last fixes found that a heading such as "### R3-F9 x" in fenced code outside ## Findings got no message: the mention check skipped it as a finding heading, and the fence skipped the check of a heading outside ## Findings. It now counts as a mention. The proof puts "## R3-F7" right after the findings, so it covers the heading that closes ## Findings too. Co-Authored-By: Claude Opus 5.5 --- scripts/template-ledger-builds-test.sh | 3 ++- sdk/extensionrepo/readiness.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/template-ledger-builds-test.sh b/scripts/template-ledger-builds-test.sh index 8d60b0a9..54f294af 100644 --- a/scripts/template-ledger-builds-test.sh +++ b/scripts/template-ledger-builds-test.sh @@ -45,13 +45,14 @@ fct_test_ledger_builds() { fct_finish_said 'this line names the severity `warn` in code, outside the line - **Severity**: of a finding' fct_finish_said 'this table names the severity warn' # A grade in prose, a finding outside ## Findings, or a severity with marks in a cell fails in any part (#420). - fct_readiness "${head}" Ready "${valid}"$'\n\n## Notes\n\n- Long lines are cut. Severity: block.\n- R3-F8 Long lines are cut: block\n\n## R3-F7 Long lines are cut\n\n## Other issues\n\n### R3-F9 Long lines are cut\n\n| R3-F9 | **Block.** |\n\n#### R3-F6 Long lines are cut' 'agent B' + fct_readiness "${head}" Ready "${valid}"$'\n\n## R3-F7 Long lines are cut\n\n## Notes\n\n- Long lines are cut. Severity: block.\n- R3-F8 Long lines are cut: block\n\n```\n### R3-F5 Long lines are cut\n```\n\n## Other issues\n\n### R3-F9 Long lines are cut\n\n| R3-F9 | **Block.** |\n\n#### R3-F6 Long lines are cut' 'agent B' fct_finish 1 fct_finish_said 'this line grades with the severity block outside the line - **Severity**: of a finding' fct_finish_said 'this finding is outside ## Findings, so finish cannot grade it' fct_finish_said 'this line names the finding R3-F8, which has no ### heading under ## Findings' fct_finish_said 'this line names the finding R3-F7, which has no ### heading under ## Findings' fct_finish_said 'this line names the finding R3-F6, which has no ### heading under ## Findings' + fct_finish_said 'this line names the finding R3-F5, which has no ### heading under ## Findings' fct_finish_said 'this table names the severity block' # Finish reads a severity only in the line - **Severity**:, so the words # block and warn pass in a title, in evidence, and in prose (#419). diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index ef10f4d5..77d4744e 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -103,7 +103,7 @@ func readinessFindings(readiness string) ([]finding, []string) { } // A heading of level 3 with an ID outside ## Findings has its own // message below, so only other lines count as a mention. - idHeading := heading != nil && len(heading[1]) == 3 && readinessID.MatchString(heading[2]) + idHeading := fence == "" && heading != nil && len(heading[1]) == 3 && readinessID.MatchString(heading[2]) if i >= 3 && (part == 0 || section) && !idHeading { for _, id := range readinessMention.FindAllString(text, -1) { if _, found := mentions[id]; !found { From e114cd1d8f8c270904f630761d501bc0f569345d Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 19:37:28 -0500 Subject: [PATCH 37/61] =?UTF-8?q?=F0=9F=9A=91=20fix:=20template:=20the=20u?= =?UTF-8?q?ser=20can=20accept=20an=20open=20finding=20at=20the=20limit=20o?= =?UTF-8?q?f=20rounds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Sol review of the stack found that an open finding could not end a build as an accepted limit. Its row under ## Open lines must name it, and any row there fails finish. Moving the row to **Limits.** changed spec.md outside **Limits.**, so finish asked for a new review of a new round, after the last one. Finish now also lets a row of ## Open lines leave when a row of **Limits.** takes its ID. Any other change of spec.md still needs a review of its commit. Refs #420 Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 1 + scripts/template-ledger-run-test.sh | 26 +++++++++++++++++++ sdk/extensionrepo/ledger_history.go | 7 ++--- sdk/extensionrepo/readiness_verdict.go | 14 ++++++++-- .../nicely-extension/readiness-review.md | 1 + 5 files changed, 44 insertions(+), 5 deletions(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index 790f8891..e4d1c81c 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -82,6 +82,7 @@ A finding of the last round can have no state yet. The history is every committed `readiness.md` of HEAD, so a moved Intake hides no finding and resets no round. Finish checks a review that ends a build with the `spec.md` of its own commit. A later `spec.md` before the next round stands in when only **Limits.** differs, so a limit committed after the review counts. +The rows of `## Open lines` whose IDs became rows of **Limits.** may leave too, so the user can accept an open finding at the limit of rounds. A limit or a spec ID that a later build changes does not change that check. A review without a ledger, from a template before the ledger, needs a passing verdict only to end a build. Its form of #409, a line `- **Limit**:` with a spec ID, reads as the state `limit`. diff --git a/scripts/template-ledger-run-test.sh b/scripts/template-ledger-run-test.sh index fe9fa854..bea7d25d 100644 --- a/scripts/template-ledger-run-test.sh +++ b/scripts/template-ledger-run-test.sh @@ -17,6 +17,32 @@ fct_test_ledger_run() { fct_finish_said 'spec.md: ## Open lines does not name the open finding' absent git -C "${finish}" checkout -q spec.md + # Round 4 stops at a limit with R4-F1 open and named under ## Open lines. + # The user accepts it, so its row moves from ## Open lines to **Limits.**, + # and the reviewer gives it the state limit with no new round. + local before stop saved="${ledger_earlier}" r4=$'### R4-F1 Long lines are cut\n\n- **Severity**: `warn`' + before="$(git -C "${finish}" rev-parse HEAD)" ledger_earlier="${round1} ${round2} ${head}" + git -C "${finish}" checkout -q readiness.md + fct_edit "${finish}/spec.md" 'None.' $'| ID | Command | Part | Case | Code |\n|---|---|---|---|---|\n| `long-lines` | inspect | Failures | R4-F1: a line over 1 MB is cut | `RUNTIME` |' + git -C "${finish}" commit -qm 'round 4 stops with an open finding' spec.md + stop="$(git -C "${finish}" rev-parse HEAD)" + fct_readiness "${stop}" 'Needs work' "${valid}"$'\n\n'"${r4}"$'\n- **State**: `open`\n- **Spec**: `long-lines`' 'agent B' + git -C "${finish}" add readiness.md + git -C "${finish}" commit -qm 'readiness review of round 4' + fct_finish 1 + fct_finish_said 'the finding R4-F1 is open' + fct_edit "${finish}/spec.md" '| `long-lines` | inspect | Failures | R4-F1: a line over 1 MB is cut | `RUNTIME` |' '' + fct_edit "${finish}/spec.md" '**Limits.** None.' $'**Limits.**\n\n| ID | State | Limit | Accepted |\n|---|---|---|---|\n| `long-lines` | `limit` | A line over 1 MB is cut | The user, 2026-10-31, https://example.com/pull/1 |' + git -C "${finish}" commit -qm 'the user accepts the limit long-lines' spec.md + fct_readiness "${stop}" 'Ready with stated limits' "${valid}"$'\n\n'"${r4}"$'\n- **State**: `limit`\n- **Spec**: `long-lines`' 'agent B' + fct_finish 0 + # Any other change of spec.md still needs a review of its commit. + fct_edit "${finish}/spec.md" '| `bad-input` | The file is missing or is no regular file, or `--timeout` is no duration | `USAGE_INVALID` |' '| `bad-input` | The file is missing | `USAGE_INVALID` |' + fct_finish 1 + fct_finish_said 'and these files changed since: spec.md' + git -C "${finish}" reset -q --hard "${before}" + ledger_earlier="${saved}" + # Round 1 records the run of its first scenario: the command, a failing # exit status, and the assertion that failed (#420). ledger_run='' fct_readiness "${head}" Ready "${valid}" 'agent B' diff --git a/sdk/extensionrepo/ledger_history.go b/sdk/extensionrepo/ledger_history.go index ddd964b8..bdcf1919 100644 --- a/sdk/extensionrepo/ledger_history.go +++ b/sdk/extensionrepo/ledger_history.go @@ -77,9 +77,10 @@ func passProblems(rounds []round, reviews []review) []string { // hold no Fill: line and no open line, and only readiness.md and **Limits.** // of spec.md differ from head. A later spec.md before the next round can // stand in when only **Limits.** differs, so a limit committed after the -// review counts. The verdict and the states follow from the findings, which -// keep each finding of the reviews before. A review without a ledger, of a -// template before it, needs no state and no history. +// review counts. The rows of ## Open lines of those limits may leave too. +// The verdict and the states follow from the findings, which keep each +// finding of the reviews before. A review without a ledger, of a template +// before it, needs no state and no history. func passedAt(reviews []review, at int, head, next string) bool { r := reviews[at] show := func(commit, file string) (string, bool) { diff --git a/sdk/extensionrepo/readiness_verdict.go b/sdk/extensionrepo/readiness_verdict.go index 71fb7009..868eca15 100644 --- a/sdk/extensionrepo/readiness_verdict.go +++ b/sdk/extensionrepo/readiness_verdict.go @@ -115,13 +115,23 @@ func limitsOnly(commit string) bool { } // sameOutsideLimits reports whether two texts of spec.md differ only under -// **Limits.**. +// **Limits.**, and under ## Open lines by the rows of the limits of b. So +// an open finding that the user accepts moves its row from ## Open lines to +// **Limits.** with no new round. Other lines of ## Open lines, such as its +// header or None., hold no open work. func sameOutsideLimits(a, b string) bool { + accepted := map[string]bool{} + for _, l := range SpecLines(b) { + if l.Part == "Limits" && l.Kind == SpecRow && l.ID != "" { + accepted[l.ID] = true + } + } outside := func(spec string) []string { lines := strings.Split(spec, "\n") var kept []string for _, l := range SpecLines(spec) { - if l.Part != "Limits" { + open := l.Kind == SpecRow || l.Kind == SpecBullet + if l.Part != "Limits" && (l.Part != "Open lines" || open && !accepted[l.ID]) { kept = append(kept, lines[l.Number-1]) } } diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 9480c63b..3351f2e7 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -72,6 +72,7 @@ End the review with one verdict. The verdict follows from the findings, and `jus Only the user accepts a limit. Ask the user, and wait for the answer. Then write the limit as a row under **Limits.** of its command, with the columns ID, State, Limit, and Accepted. +An open finding moves its row from `## Open lines` to that table, with the same ID. This change starts no round: the reviewer gives the finding the state `limit` in the review of the same commit. Commit the limit in `spec.md`, then commit `readiness.md` alone. Its State is `limit`. Accepted records who accepted it, when, and where, such as a review comment. From b6e5e21dce21ac64957aad99e99eb07bee396057 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:16:13 -0400 Subject: [PATCH 38/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20sdk/exte?= =?UTF-8?q?nsiontest:=20the=20answer=20schema=20check=20moves=20to=20its?= =?UTF-8?q?=20own=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validate and noRemote move from scenario.go to answer_schema.go, with no change of behavior, so a fix of the schema name can grow them. Co-Authored-By: Claude Opus 5.5 --- sdk/extensiontest/answer_schema.go | 43 ++++++++++++++++++++++++++++++ sdk/extensiontest/scenario.go | 35 ------------------------ 2 files changed, 43 insertions(+), 35 deletions(-) create mode 100644 sdk/extensiontest/answer_schema.go diff --git a/sdk/extensiontest/answer_schema.go b/sdk/extensiontest/answer_schema.go new file mode 100644 index 00000000..a39d8958 --- /dev/null +++ b/sdk/extensiontest/answer_schema.go @@ -0,0 +1,43 @@ +package extensiontest + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +// validate checks an answer against a schema of draft 2020-12, with a +// loader that refuses every remote schema. +func validate(schema map[string]any, answer string) error { + raw, err := json.Marshal(schema) + if err != nil { + return err + } + doc, err := jsonschema.UnmarshalJSON(strings.NewReader(string(raw))) + if err != nil { + return err + } + compiler := jsonschema.NewCompiler() + compiler.DefaultDraft(jsonschema.Draft2020) + compiler.UseLoader(noRemote{}) + if err := compiler.AddResource("answer.json", doc); err != nil { + return err + } + compiled, err := compiler.Compile("answer.json") + if err != nil { + return err + } + value, err := jsonschema.UnmarshalJSON(strings.NewReader(answer)) + if err != nil { + return fmt.Errorf("not JSON: %w", err) + } + return compiled.Validate(value) +} + +// noRemote refuses to load any schema, so a reference outside the schema +// fails instead of reaching the network. +type noRemote struct{} + +func (noRemote) Load(url string) (any, error) { return nil, fmt.Errorf("no remote schema: %s", url) } diff --git a/sdk/extensiontest/scenario.go b/sdk/extensiontest/scenario.go index 04f7ad9e..6eb01070 100644 --- a/sdk/extensiontest/scenario.go +++ b/sdk/extensiontest/scenario.go @@ -18,7 +18,6 @@ import ( "github.com/pascalandy/nicely/sdk/command" "github.com/pascalandy/nicely/sdk/contract" "github.com/rogpeppe/go-internal/testscript" - "github.com/santhosh-tekuri/jsonschema/v6" "github.com/spf13/cobra" ) @@ -239,37 +238,3 @@ func (k *Coverage) find(words []string) (parsed, bool) { call.runs = found && k.manifest.Runs(call.command) return call, true } - -// validate checks an answer against a schema of draft 2020-12, with a -// loader that refuses every remote schema. -func validate(schema map[string]any, answer string) error { - raw, err := json.Marshal(schema) - if err != nil { - return err - } - doc, err := jsonschema.UnmarshalJSON(strings.NewReader(string(raw))) - if err != nil { - return err - } - compiler := jsonschema.NewCompiler() - compiler.DefaultDraft(jsonschema.Draft2020) - compiler.UseLoader(noRemote{}) - if err := compiler.AddResource("answer.json", doc); err != nil { - return err - } - compiled, err := compiler.Compile("answer.json") - if err != nil { - return err - } - value, err := jsonschema.UnmarshalJSON(strings.NewReader(answer)) - if err != nil { - return fmt.Errorf("not JSON: %w", err) - } - return compiled.Validate(value) -} - -// noRemote refuses to load any schema, so a reference outside the schema -// fails instead of reaching the network. -type noRemote struct{} - -func (noRemote) Load(url string) (any, error) { return nil, fmt.Errorf("no remote schema: %s", url) } From 1939de63c6e376c66ed603f70172c9d5af112ecb Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:16:34 -0400 Subject: [PATCH 39/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20sdk/cont?= =?UTF-8?q?ract:=20the=20schema=20compiler=20moves=20to=20its=20own=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit compileSchema and noRemote move from schema.go to schema_compile.go, with no change of behavior, so a fix of the schema name can grow them. Co-Authored-By: Claude Opus 5.5 --- sdk/contract/schema.go | 33 ----------------------------- sdk/contract/schema_compile.go | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 33 deletions(-) create mode 100644 sdk/contract/schema_compile.go diff --git a/sdk/contract/schema.go b/sdk/contract/schema.go index fd55097f..eedda82b 100644 --- a/sdk/contract/schema.go +++ b/sdk/contract/schema.go @@ -1,16 +1,12 @@ package contract import ( - "bytes" - "encoding/json" "errors" "maps" "net/url" "slices" "strconv" "strings" - - "github.com/santhosh-tekuri/jsonschema/v6" ) // SchemaDialect is the JSON Schema draft of every schema that ncly @@ -50,35 +46,6 @@ func ResolveSchema(schema map[string]any) (map[string]any, error) { return root, nil } -// compileSchema checks a schema against draft 2020-12, with a loader that -// refuses every remote schema. -func compileSchema(schema map[string]any) error { - data, err := json.Marshal(schema) - if err != nil { - return err - } - doc, err := jsonschema.UnmarshalJSON(bytes.NewReader(data)) - if err != nil { - return err - } - c := jsonschema.NewCompiler() - c.DefaultDraft(jsonschema.Draft2020) - c.UseLoader(noRemote{}) - if err := c.AddResource("schema.json", doc); err != nil { - return err - } - _, err = c.Compile("schema.json") - return err -} - -// noRemote refuses to load a schema, so that a reference outside the -// schema fails instead of reaching the network. -type noRemote struct{} - -func (noRemote) Load(url string) (any, error) { - return nil, errors.New("schema: no remote schema: " + url) -} - // Keywords of draft 2020-12 that hold schemas: one schema, a map of // schemas, or a list of schemas. Every other keyword holds a literal value. var ( diff --git a/sdk/contract/schema_compile.go b/sdk/contract/schema_compile.go new file mode 100644 index 00000000..05835415 --- /dev/null +++ b/sdk/contract/schema_compile.go @@ -0,0 +1,38 @@ +package contract + +import ( + "bytes" + "encoding/json" + "errors" + + "github.com/santhosh-tekuri/jsonschema/v6" +) + +// compileSchema checks a schema against draft 2020-12, with a loader that +// refuses every remote schema. +func compileSchema(schema map[string]any) error { + data, err := json.Marshal(schema) + if err != nil { + return err + } + doc, err := jsonschema.UnmarshalJSON(bytes.NewReader(data)) + if err != nil { + return err + } + c := jsonschema.NewCompiler() + c.DefaultDraft(jsonschema.Draft2020) + c.UseLoader(noRemote{}) + if err := c.AddResource("schema.json", doc); err != nil { + return err + } + _, err = c.Compile("schema.json") + return err +} + +// noRemote refuses to load a schema, so that a reference outside the +// schema fails instead of reaching the network. +type noRemote struct{} + +func (noRemote) Load(url string) (any, error) { + return nil, errors.New("schema: no remote schema: " + url) +} From 491f1da208b0a5dd81a1f5bc97b6d28d30dac6b2 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:16:45 -0400 Subject: [PATCH 40/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20contract:=20ncly=20?= =?UTF-8?q?starts=20in=20a=20folder=20whose=20path=20holds=20a=20percent?= =?UTF-8?q?=20sign?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The schema compiler resolved the relative name schema.json against the working folder as a file URL. A % in that path made ncly panic at startup with "invalid URL escape", even for --version. The schemas now have a URN that no folder changes, and so do the answers that the test kit checks. Refs #423 Co-Authored-By: Claude Opus 5.5 --- sdk/contract/parity_test.go | 4 ++-- sdk/contract/schema_compile.go | 9 +++++++-- sdk/extensiontest/answer_schema.go | 7 ++++--- testdata/script/version.txtar | 7 +++++++ 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/sdk/contract/parity_test.go b/sdk/contract/parity_test.go index 4fb64f9d..1ed73c04 100644 --- a/sdk/contract/parity_test.go +++ b/sdk/contract/parity_test.go @@ -179,10 +179,10 @@ func compileAnswerSchema(t *testing.T, kind ResultKind, data map[string]any) *js c := jsonschema.NewCompiler() c.DefaultDraft(jsonschema.Draft2020) c.UseLoader(noRemote{}) - if err := c.AddResource("answer.json", doc); err != nil { + if err := c.AddResource(schemaURL, doc); err != nil { t.Fatal(err) } - schema, err := c.Compile("answer.json") + schema, err := c.Compile(schemaURL) if err != nil { t.Fatal(err) } diff --git a/sdk/contract/schema_compile.go b/sdk/contract/schema_compile.go index 05835415..59556612 100644 --- a/sdk/contract/schema_compile.go +++ b/sdk/contract/schema_compile.go @@ -8,6 +8,11 @@ import ( "github.com/santhosh-tekuri/jsonschema/v6" ) +// schemaURL names the schema that compileSchema checks. A relative name +// would resolve against the working folder as a file URL, and a % in its +// path stopped ncly at startup (#423). +const schemaURL = "urn:nicely:schema.json" + // compileSchema checks a schema against draft 2020-12, with a loader that // refuses every remote schema. func compileSchema(schema map[string]any) error { @@ -22,10 +27,10 @@ func compileSchema(schema map[string]any) error { c := jsonschema.NewCompiler() c.DefaultDraft(jsonschema.Draft2020) c.UseLoader(noRemote{}) - if err := c.AddResource("schema.json", doc); err != nil { + if err := c.AddResource(schemaURL, doc); err != nil { return err } - _, err = c.Compile("schema.json") + _, err = c.Compile(schemaURL) return err } diff --git a/sdk/extensiontest/answer_schema.go b/sdk/extensiontest/answer_schema.go index a39d8958..6c978dfa 100644 --- a/sdk/extensiontest/answer_schema.go +++ b/sdk/extensiontest/answer_schema.go @@ -9,7 +9,8 @@ import ( ) // validate checks an answer against a schema of draft 2020-12, with a -// loader that refuses every remote schema. +// loader that refuses every remote schema. The URN keeps the working folder +// out of the name of the schema, as in contract.compileSchema (#423). func validate(schema map[string]any, answer string) error { raw, err := json.Marshal(schema) if err != nil { @@ -22,10 +23,10 @@ func validate(schema map[string]any, answer string) error { compiler := jsonschema.NewCompiler() compiler.DefaultDraft(jsonschema.Draft2020) compiler.UseLoader(noRemote{}) - if err := compiler.AddResource("answer.json", doc); err != nil { + if err := compiler.AddResource("urn:nicely:answer.json", doc); err != nil { return err } - compiled, err := compiler.Compile("answer.json") + compiled, err := compiler.Compile("urn:nicely:answer.json") if err != nil { return err } diff --git a/testdata/script/version.txtar b/testdata/script/version.txtar index 90bcb226..03c56a22 100644 --- a/testdata/script/version.txtar +++ b/testdata/script/version.txtar @@ -20,3 +20,10 @@ stdout '\Ancly v\S+\n\z' # --help still wins over --version exec ncly --version --help stdout 'Curated CLI toolbox' + +# A percent sign in the path of the working folder changes nothing (#423) +mkdir 'work 100%' +cd 'work 100%' +exec ncly --version +stdout '\Ancly v\S+\n\z' +! stderr . From bdd448519a5c3b14238f43a6ec286c1d6b052c88 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:17:39 -0400 Subject: [PATCH 41/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20template?= =?UTF-8?q?:=20the=20severity=20forms=20of=20finish=20move=20to=20their=20?= =?UTF-8?q?own=20files?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The severity patterns, severityCause, and severityCell move from readiness.go to readiness_severity.go. Their proofs move from template-finish-test.sh to template-finish-forms-test.sh. Nothing changes in behavior, so the next fix can grow them. Co-Authored-By: Claude Opus 5.5 --- scripts/template-finish-forms-test.sh | 42 +++++++++++++++++++ scripts/template-finish-test.sh | 34 ++------------- sdk/extensionrepo/readiness.go | 43 ------------------- sdk/extensionrepo/readiness_severity.go | 55 +++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 74 deletions(-) create mode 100644 scripts/template-finish-forms-test.sh create mode 100644 sdk/extensionrepo/readiness_severity.go diff --git a/scripts/template-finish-forms-test.sh b/scripts/template-finish-forms-test.sh new file mode 100644 index 00000000..6a0fbef2 --- /dev/null +++ b/scripts/template-finish-forms-test.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Sourced by template-finish-test.sh. fct_test_finish calls +# fct_test_finish_forms with its repository and its fixture variables. It +# proves that just finish fails on each finding that it cannot grade, and on +# each form that grades outside the line - **Severity**: of a finding. +# Markdown backticks in the fixture text are literal. +# shellcheck disable=SC2016,SC2154 + +fct_test_finish_forms() { + local number + local hidden=$'### R1-F2 The count could name the longest line\n\n- **Severity**: `advise`\n\n| Output keeps the author | `warn` |\n| Output keeps the serial number | block |\n\n```\n### Output keeps the place\n\n- **Severity**: `warn`\n```\n\n### R1-F3 Output keeps the name\n\n- Severity: warn\n- **Severity**: `advise`\n\n### R1-F4 Output keeps the date\n\nSeverity: warn.\n\n- **Severity**: `advise`\n\n### Severity: warn\n\n- **Severity**: `advise`\n- **Spec**: `block`' + local unread=$'- **Severity**: `warn`\n\n### No severity\n\n### Two severities\n\n- **Severity**: `advise`\n- **Severity**: `warn`\n\n### Unknown severity\n\n- **Severity**: minor\n\n### A fenced heading\n\n````\n# a comment\n```\n````\n\n- **Severity**: `block`\n\n### An open fence\n\n```\n- **Severity**: `advise`' + # A finding that finish cannot grade fails it. + fct_readiness "${sha}" 'Ready with stated limits' '' + fct_finish 1 + fct_finish_said 'readiness.md: it holds no part ## Findings' + fct_readiness "${sha}" 'Ready with stated limits' $'## Verdict\n\nReady with stated limits.' + fct_finish 1 + fct_finish_said 'readiness.md:7: ## Findings holds no ### finding and no None.' + fct_readiness "${sha}" 'Ready with stated limits' $'| Finding | Severity |\n|---|---|\n| Output is lost | `block` |\n\n'"${advise}" + fct_finish 1 + fct_finish_said 'readiness.md:9: this text of ## Findings is outside every ### finding' + fct_readiness "${sha}" 'Ready with stated limits' "${unread}" + fct_finish 1 + fct_finish_said 'readiness.md:9: this severity has no finding of its own' + fct_finish_said 'readiness.md:11: this finding has no severity' + fct_finish_said 'readiness.md:16: this finding already has a severity' + fct_finish_said 'readiness.md:20: the severity "minor" is not block, warn, or advise' + fct_finish_said 'readiness.md:22: the verdict is Ready with stated limits, but this finding has the severity block' + fct_finish_said 'readiness.md:33: this fenced code has no end' + + # A severity outside the line - **Severity**: of a finding fails it, even after a valid finding (#413). + fct_readiness "${sha}" 'Ready with stated limits' "${hidden}" + fct_finish 1 + fct_finish_said 'readiness.md:13: this table names the severity warn' + fct_finish_said 'readiness.md:14: this table names the severity block' + fct_finish_said 'readiness.md:19: this fenced code names the severity `warn`' + for number in 24 29 33; do + fct_finish_said "readiness.md:${number}: this line grades with the severity warn" + done + fct_finish_said 'readiness.md:36: this line names the severity `block` in code' +} diff --git a/scripts/template-finish-test.sh b/scripts/template-finish-test.sh index 6895b311..e7c9aeca 100644 --- a/scripts/template-finish-test.sh +++ b/scripts/template-finish-test.sh @@ -7,6 +7,8 @@ # shellcheck source=scripts/template-finish-helpers.sh source "${BASH_SOURCE[0]%/*}/template-finish-helpers.sh" +# shellcheck source=scripts/template-finish-forms-test.sh +source "${BASH_SOURCE[0]%/*}/template-finish-forms-test.sh" fct_test_finish() { local finish="${2}/finish" scratch="${2}" sha file number ledger_earlier="" @@ -14,8 +16,6 @@ fct_test_finish() { local unfilled='this line of the form is not filled' local warn=$'### R1-F1 Large files are slow\n\n- **Severity**: `warn`' local advise=$'### R1-F2 The count could name the longest line\n\n- **Severity**: `advise`' - local hidden=$'### R1-F2 The count could name the longest line\n\n- **Severity**: `advise`\n\n| Output keeps the author | `warn` |\n| Output keeps the serial number | block |\n\n```\n### Output keeps the place\n\n- **Severity**: `warn`\n```\n\n### R1-F3 Output keeps the name\n\n- Severity: warn\n- **Severity**: `advise`\n\n### R1-F4 Output keeps the date\n\nSeverity: warn.\n\n- **Severity**: `advise`\n\n### Severity: warn\n\n- **Severity**: `advise`\n- **Spec**: `block`' - local unread=$'- **Severity**: `warn`\n\n### No severity\n\n### Two severities\n\n- **Severity**: `advise`\n- **Severity**: `warn`\n\n### Unknown severity\n\n- **Severity**: minor\n\n### A fenced heading\n\n````\n# a comment\n```\n````\n\n- **Severity**: `block`\n\n### An open fence\n\n```\n- **Severity**: `advise`' cp -R "${1}" "${finish}" fct_steps "${finish}" [[ "$(just --justfile "${finish}/justfile" --show finish | sed '/^#/d')" == $'finish:\n go run github.com/pascalandy/nicely/sdk/extensionrepo/cmd/extensionrepo finish' ]] || @@ -98,35 +98,7 @@ fct_test_finish() { fct_finish 1 fct_finish_said 'readiness.md: Git knows no commit 0123456789abcdef0123456789abcdef01234567' - # A finding that finish cannot grade fails it. - fct_readiness "${sha}" 'Ready with stated limits' '' - fct_finish 1 - fct_finish_said 'readiness.md: it holds no part ## Findings' - fct_readiness "${sha}" 'Ready with stated limits' $'## Verdict\n\nReady with stated limits.' - fct_finish 1 - fct_finish_said 'readiness.md:7: ## Findings holds no ### finding and no None.' - fct_readiness "${sha}" 'Ready with stated limits' $'| Finding | Severity |\n|---|---|\n| Output is lost | `block` |\n\n'"${advise}" - fct_finish 1 - fct_finish_said 'readiness.md:9: this text of ## Findings is outside every ### finding' - fct_readiness "${sha}" 'Ready with stated limits' "${unread}" - fct_finish 1 - fct_finish_said 'readiness.md:9: this severity has no finding of its own' - fct_finish_said 'readiness.md:11: this finding has no severity' - fct_finish_said 'readiness.md:16: this finding already has a severity' - fct_finish_said 'readiness.md:20: the severity "minor" is not block, warn, or advise' - fct_finish_said 'readiness.md:22: the verdict is Ready with stated limits, but this finding has the severity block' - fct_finish_said 'readiness.md:33: this fenced code has no end' - - # A severity outside the line - **Severity**: of a finding fails it, even after a valid finding (#413). - fct_readiness "${sha}" 'Ready with stated limits' "${hidden}" - fct_finish 1 - fct_finish_said 'readiness.md:13: this table names the severity warn' - fct_finish_said 'readiness.md:14: this table names the severity block' - fct_finish_said 'readiness.md:19: this fenced code names the severity `warn`' - for number in 24 29 33; do - fct_finish_said "readiness.md:${number}: this line grades with the severity warn" - done - fct_finish_said 'readiness.md:36: this line names the severity `block` in code' + fct_test_finish_forms # Finish reads Commit:, Reviewer:, and Verdict: only in the first lines, never in fenced code (#413). fct_readiness "${sha}" 'Ready with stated limits' $'```\nReviewer: an agent that wrote none of the code\n```\n\nNone.' '' diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index 77d4744e..a2de9750 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -27,51 +27,8 @@ var ( readinessID = regexp.MustCompile(`^(R([1-9][0-9]*)-F[1-9][0-9]*)\b`) // readinessMention finds the ID of a finding anywhere in a line. readinessMention = regexp.MustCompile(`\bR[1-9][0-9]*-F[1-9][0-9]*\b`) - // readinessSeverity finds a severity in code, a form that finish cannot - // grade, anywhere after the header (#413, #416). The plain words pass, so - // a title or prose can say that a call blocks or warns (#419). - readinessSeverity = regexp.MustCompile("(?i)`(block|warn|advise)`") - // readinessGrade finds the word severity before a severity, a grade in - // prose such as "Severity: block", in any part (#420). - readinessGrade = regexp.MustCompile(`(?i)\bseverity\b[^a-z0-9]*\b(block|warn|advise)\b`) - // readinessWord finds the word severity under ## Findings (#413). - readinessWord = regexp.MustCompile(`(?i)\bseverity\b`) ) -// severityCause names why a line that names a severity fails finish, so the -// reviewer sees the word or the table that caused it (#416). -func severityCause(text, fence string) string { - const fix = "Fix: grade each finding only in its line - **Severity**:, and write " - code, cell, grade := readinessSeverity.FindStringSubmatch(text), severityCell(text), readinessGrade.FindStringSubmatch(text) - switch { - case cell != "": - return fmt.Sprintf("this table names the severity %s. Fix: remove the table, because each finding gives its severity in its line - **Severity**:", cell) - case code != nil && fence != "": - return fmt.Sprintf("this fenced code names the severity `%s`. %sthe word %s in other text without backticks", strings.ToLower(code[1]), fix, strings.ToLower(code[1])) - case code != nil: - return fmt.Sprintf("this line names the severity `%s` in code, outside the line - **Severity**: of a finding. %sthe word %s in other text without backticks", strings.ToLower(code[1]), fix, strings.ToLower(code[1])) - case grade != nil: - return fmt.Sprintf("this line grades with the severity %s outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity", strings.ToLower(grade[1])) - } - return "this line names the word severity outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity" -} - -// severityCell returns the severity that a cell of a table row holds alone, -// with only marks around it, such as `warn`, **block**, or (advise). It -// returns "" for any other line (#416, #420). -func severityCell(text string) string { - trimmed := strings.TrimSpace(text) - if !strings.HasPrefix(trimmed, "|") { - return "" - } - for _, cell := range strings.Split(trimmed, "|") { - if word := strings.ToLower(strings.Trim(cell, " \t`*_().:!")); severityRank[word] > 0 { - return word - } - } - return "" -} - // readinessFindings reads the findings of readiness.md, outside fenced code. // It reports each finding that finish cannot grade, so a finding in another // form never passes as no finding. Under ## Findings, it also reports each diff --git a/sdk/extensionrepo/readiness_severity.go b/sdk/extensionrepo/readiness_severity.go new file mode 100644 index 00000000..fef85c3f --- /dev/null +++ b/sdk/extensionrepo/readiness_severity.go @@ -0,0 +1,55 @@ +package extensionrepo + +import ( + "fmt" + "regexp" + "strings" +) + +// The forms of a severity that finish refuses outside the line +// - **Severity**: of a finding. +var ( + // readinessSeverity finds a severity in code, a form that finish cannot + // grade, anywhere after the header (#413, #416). The plain words pass, so + // a title or prose can say that a call blocks or warns (#419). + readinessSeverity = regexp.MustCompile("(?i)`(block|warn|advise)`") + // readinessGrade finds the word severity before a severity, a grade in + // prose such as "Severity: block", in any part (#420). + readinessGrade = regexp.MustCompile(`(?i)\bseverity\b[^a-z0-9]*\b(block|warn|advise)\b`) + // readinessWord finds the word severity under ## Findings (#413). + readinessWord = regexp.MustCompile(`(?i)\bseverity\b`) +) + +// severityCause names why a line that names a severity fails finish, so the +// reviewer sees the word or the table that caused it (#416). +func severityCause(text, fence string) string { + const fix = "Fix: grade each finding only in its line - **Severity**:, and write " + code, cell, grade := readinessSeverity.FindStringSubmatch(text), severityCell(text), readinessGrade.FindStringSubmatch(text) + switch { + case cell != "": + return fmt.Sprintf("this table names the severity %s. Fix: remove the table, because each finding gives its severity in its line - **Severity**:", cell) + case code != nil && fence != "": + return fmt.Sprintf("this fenced code names the severity `%s`. %sthe word %s in other text without backticks", strings.ToLower(code[1]), fix, strings.ToLower(code[1])) + case code != nil: + return fmt.Sprintf("this line names the severity `%s` in code, outside the line - **Severity**: of a finding. %sthe word %s in other text without backticks", strings.ToLower(code[1]), fix, strings.ToLower(code[1])) + case grade != nil: + return fmt.Sprintf("this line grades with the severity %s outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity", strings.ToLower(grade[1])) + } + return "this line names the word severity outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity" +} + +// severityCell returns the severity that a cell of a table row holds alone, +// with only marks around it, such as `warn`, **block**, or (advise). It +// returns "" for any other line (#416, #420). +func severityCell(text string) string { + trimmed := strings.TrimSpace(text) + if !strings.HasPrefix(trimmed, "|") { + return "" + } + for _, cell := range strings.Split(trimmed, "|") { + if word := strings.ToLower(strings.Trim(cell, " \t`*_().:!")); severityRank[word] > 0 { + return word + } + } + return "" +} From 7afe8ba7079fa39892013e50e31dd9e1f72eb183 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:18:27 -0400 Subject: [PATCH 42/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20finish?= =?UTF-8?q?=20fails=20on=20a=20grade=20in=20free=20form,=20and=20states=20?= =?UTF-8?q?its=20limit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A review could grade in notes and pass just finish, with forms such as "Grade: block", "warn-level", or the cell "really a warn". Finish now fails on the word grade before a severity, on a severity before the word level, and on a cell whose last word is a severity. Ordinary prose with the words block and warn still passes. finish-gate.md states the exact forms that the parser finds, and the reviewer moves each other grade to the bullet Severity. Refs #422, #423 Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 16 +++++++++- scripts/template-finish-forms-test.sh | 7 +++++ scripts/template-finish-test.sh | 2 +- sdk/extensionrepo/readiness.go | 2 +- sdk/extensionrepo/readiness_severity.go | 30 +++++++++++++------ .../nicely-extension/readiness-review.md | 7 +++-- template/AGENTS/rule-index.md.jinja | 2 ++ 7 files changed, 52 insertions(+), 14 deletions(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index e4d1c81c..a8d51b41 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -12,7 +12,7 @@ This part says what `just finish` checks in an extension repository, as [Extensi - A `readiness.md` whose first three lines hold no `Commit: `, no `Reviewer: `, or no verdict `Ready` or `Ready with stated limits`. A byte order mark before them is ignored - A reviewed commit that differs from the working tree outside `readiness.md`, and outside the part **Limits.** of `spec.md`. So the user accepts a limit after a review, with no new round - A finding of `readiness.md` without its own `###` heading under `## Findings`, or without one severity: `block`, `warn`, or `advise` -- A severity after the first three lines, outside the line `- **Severity**:` of a finding, in a form that grades. Such a form is code, such as `` `warn` ``, also in fenced code. A cell of a table that holds only a severity is such a form too, also with marks such as `**` around it. The word severity before a severity, such as "Severity: block", is a form that grades too. Under `## Findings`, the word severity fails alone. The message names the word or the table that caused it +- A severity after the first three lines, outside the line `- **Severity**:` of a finding, in a form that grades. [The limit of the parser](#the-limit-of-the-parser) lists each such form. The message names the word or the table that caused it - A finding heading with an ID, such as `### R1-F2`, outside `## Findings`. Also a line outside `## Findings` that names the ID of no finding under it - `Ready` with a `block` or `warn` finding that is not `fixed`, or with a limit in `spec.md` - `Ready with stated limits` with a `block` finding that is not `fixed`, or with a `warn` that is neither `fixed` nor an accepted `limit` @@ -21,6 +21,20 @@ Finish reads a severity only in the line `- **Severity**:` of a finding. The plain words block, warn, and advise pass in a title, in evidence, and in prose (#419). An example is the title `### R1-F1 Large inputs block the terminal`. +### The limit of the parser + +The severities are the words block, warn, and advise, in any case. +After the first three lines, outside the line `- **Severity**:` of a finding, finish fails on these forms, and only on these (#413, #416, #420, #422): + +- A severity in code, such as `` `warn` ``, also in fenced code +- A cell of a table whose last word is a severity, with only marks after it. Examples are `` `warn` ``, `**block**`, `(advise)`, and `really a warn` +- The word severity, grade, grades, or graded, then only marks or spaces, then a severity, such as "Severity: block" or "Grade: warn" +- A severity, then a hyphen or a space, then the word level, such as "warn-level" +- Under `## Findings`, the word severity alone + +Finish reads each other line as prose, and it passes. This is also true when the line grades in other words, such as "the grade is warn" or "a block for me". +The reviewer gives each grade only in the bullet **Severity** of its finding, and moves each other grade there before the verdict. + ## The ledger `readiness.md` also holds the ledger of the repository (#417). diff --git a/scripts/template-finish-forms-test.sh b/scripts/template-finish-forms-test.sh index 6a0fbef2..acac596f 100644 --- a/scripts/template-finish-forms-test.sh +++ b/scripts/template-finish-forms-test.sh @@ -39,4 +39,11 @@ fct_test_finish_forms() { fct_finish_said "readiness.md:${number}: this line grades with the severity warn" done fct_finish_said 'readiness.md:36: this line names the severity `block` in code' + + # A grade in a free form, such as in notes, fails too (#422). + fct_readiness "${sha}" 'Ready with stated limits' "${warn}"$'\n- **State**: `limit`\n- **Spec**: `large-file`\n\n'"${advise}"$'\n\n## Notes\n\n| R1-F1 | really a warn |\n\nGrade: block. The name in the output is a block for me.\n\nThis issue is warn-level and should block the release.' + fct_finish 1 + fct_finish_said 'readiness.md:21: this table names the severity warn' + fct_finish_said 'readiness.md:23: this line grades with the severity block outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word grade' + fct_finish_said 'readiness.md:25: this line grades with the severity warn outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the form warn-level' } diff --git a/scripts/template-finish-test.sh b/scripts/template-finish-test.sh index e7c9aeca..b118036a 100644 --- a/scripts/template-finish-test.sh +++ b/scripts/template-finish-test.sh @@ -108,7 +108,7 @@ fct_test_finish() { fct_finish 1 fct_finish_said 'readiness.md: its first three lines hold no line Commit: or Verdict: ' - fct_readiness "${sha}" 'Ready with stated limits' "${warn}"$'\n- **State**: `limit`\n- **Spec**: `large-file`\n\n'"${advise}"$'\n- **State**: `declined`\n- **Reason**: the answer stays small\n\n## Severity assessment\n\nEach finding names its severity.' + fct_readiness "${sha}" 'Ready with stated limits' "${warn}"$'\n- **State**: `limit`\n- **Spec**: `large-file`\n\n'"${advise}"$'\n- **State**: `declined`\n- **Reason**: the answer stays small\n\n## Severity assessment\n\nEach finding names its severity.\n\n## Notes\n\n| R1-F2 | the answer stays small |\n\nLarge files block the terminal, and inspect may warn.' git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'readiness review' fct_finish 0 diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index a2de9750..846b2613 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -54,7 +54,7 @@ func readinessFindings(readiness string) ([]finding, []string) { bullet := part > 0 && fence == "" && m != nil && m[1] == "Severity" // The first three lines are the header. A severity elsewhere fails, // also under another part after ## Findings (#416). - if i >= 3 && !bullet && (readinessSeverity.MatchString(text) || severityCell(text) != "" || readinessGrade.MatchString(text) || + if i >= 3 && !bullet && (readinessSeverity.MatchString(text) || severityCell(text) != "" || readinessGrade.MatchString(text) || readinessLevel.MatchString(text) || part > 0 && !section && readinessWord.MatchString(text)) { problems = append(problems, fmt.Sprintf("readiness.md:%d: %s", i+1, severityCause(text, fence))) } diff --git a/sdk/extensionrepo/readiness_severity.go b/sdk/extensionrepo/readiness_severity.go index fef85c3f..14d2aa27 100644 --- a/sdk/extensionrepo/readiness_severity.go +++ b/sdk/extensionrepo/readiness_severity.go @@ -13,9 +13,13 @@ var ( // grade, anywhere after the header (#413, #416). The plain words pass, so // a title or prose can say that a call blocks or warns (#419). readinessSeverity = regexp.MustCompile("(?i)`(block|warn|advise)`") - // readinessGrade finds the word severity before a severity, a grade in - // prose such as "Severity: block", in any part (#420). - readinessGrade = regexp.MustCompile(`(?i)\bseverity\b[^a-z0-9]*\b(block|warn|advise)\b`) + // readinessGrade finds the word severity or grade before a severity, in + // any part. Only marks can stand between them, such as in "Severity: + // block" or "Grade: warn" (#420, #422). + readinessGrade = regexp.MustCompile(`(?i)\b(severity|grade[sd]?)\b[^a-z0-9]*\b(block|warn|advise)\b`) + // readinessLevel finds a severity before the word level, such as + // "warn-level" or "block level", in any part (#422). + readinessLevel = regexp.MustCompile(`(?i)\b(block|warn|advise)[- ]level\b`) // readinessWord finds the word severity under ## Findings (#413). readinessWord = regexp.MustCompile(`(?i)\bseverity\b`) ) @@ -24,7 +28,8 @@ var ( // reviewer sees the word or the table that caused it (#416). func severityCause(text, fence string) string { const fix = "Fix: grade each finding only in its line - **Severity**:, and write " - code, cell, grade := readinessSeverity.FindStringSubmatch(text), severityCell(text), readinessGrade.FindStringSubmatch(text) + code, cell := readinessSeverity.FindStringSubmatch(text), severityCell(text) + grade, level := readinessGrade.FindStringSubmatch(text), readinessLevel.FindStringSubmatch(text) switch { case cell != "": return fmt.Sprintf("this table names the severity %s. Fix: remove the table, because each finding gives its severity in its line - **Severity**:", cell) @@ -33,21 +38,28 @@ func severityCause(text, fence string) string { case code != nil: return fmt.Sprintf("this line names the severity `%s` in code, outside the line - **Severity**: of a finding. %sthe word %s in other text without backticks", strings.ToLower(code[1]), fix, strings.ToLower(code[1])) case grade != nil: - return fmt.Sprintf("this line grades with the severity %s outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity", strings.ToLower(grade[1])) + return fmt.Sprintf("this line grades with the severity %s outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word %s", strings.ToLower(grade[2]), strings.ToLower(grade[1])) + case level != nil: + return fmt.Sprintf("this line grades with the severity %s outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the form %s", strings.ToLower(level[1]), level[0]) } return "this line names the word severity outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity" } -// severityCell returns the severity that a cell of a table row holds alone, -// with only marks around it, such as `warn`, **block**, or (advise). It -// returns "" for any other line (#416, #420). +// severityCell returns the severity that ends a cell of a table row, with +// only marks after it, such as `warn`, **block**, or really a warn. It +// returns "" for any other line. So the cell "inputs block the terminal" +// passes (#416, #420, #422). func severityCell(text string) string { trimmed := strings.TrimSpace(text) if !strings.HasPrefix(trimmed, "|") { return "" } for _, cell := range strings.Split(trimmed, "|") { - if word := strings.ToLower(strings.Trim(cell, " \t`*_().:!")); severityRank[word] > 0 { + words := strings.Fields(strings.ToLower(cell)) + if len(words) == 0 { + continue + } + if word := strings.Trim(words[len(words)-1], "`*_().:!"); severityRank[word] > 0 { return word } } diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 3351f2e7..dc17d901 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -40,12 +40,15 @@ Name a severity only in the bullet **Severity** of its finding. `just finish` re After the first three lines, it fails on each form that grades in another place: - A severity in code, such as `` `warn` `` -- A cell of a table that holds only a severity -- The word severity before a severity, such as "Severity: block" +- A cell of a table whose last word is a severity, such as `really a warn` +- The word severity or grade before a severity, such as "Severity: block" or "Grade: warn" +- A severity before the word level, such as "warn-level" - A finding heading, such as `### R1-F2`, outside `## Findings`, or an ID outside it of no finding under it - The word severity under `## Findings` The plain words pass in a title, in evidence, and in prose, such as `### R1-F1 Large inputs block the terminal`. +`just finish` does not find a grade in other words, such as "a block for me". "The limit of the parser" in `finish-gate.md` gives the exact forms. +Before the verdict, read the whole review, also the notes. Move each grade in another form to the bullet **Severity** of its finding. A loss that a user can see is `warn` or `block`, never `advise`. Examples are data that the output keeps or changes, and a promise of `spec.md` that does not hold. diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index 3d116f8b..317223da 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -42,6 +42,8 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | `## Open lines` of `spec.md` names each finding that the ledger records as `open`, by its ID | [Ledger findings][ledger-findings] | | A severity is read only in the line **Severity** of a finding; the plain words block, warn, and advise pass elsewhere | [Readiness findings][readiness] | | A finding heading stays under `## Findings`, and no other form grades | [Readiness findings][readiness] | +| Finish finds a grade in code, at the end of a cell, after severity or grade, or before level | [Readiness findings][readiness] | +| The reviewer moves each grade in another form, which the parser does not find, to the bullet **Severity** | Prose only | | A broken promise of `spec.md` or of the help text is at least `warn` at its first review | Prose only | | Each round keeps its row and has a committed review; a build ends only at a review that passes | [Ledger history][ledger-history] | | Accepting a limit starts no round; after a review, only **Limits.** of `spec.md` changes | [Finish gate][finish-verdict] | From 59d33f385abff52b92f5ae9e1afed7f58a89daa4 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:18:29 -0400 Subject: [PATCH 43/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20a=20fin?= =?UTF-8?q?ding=20of=20this=20round=20with=20no=20state=20is=20open?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A review could add a finding with no state beside None. under ## Open lines, and just finish passed when the verdict allowed it. Such a finding of the last round is now open: finish fails on it, and fails with its ID when ## Open lines omits it. Refs #423 Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 6 ++--- scripts/template-finish-test.sh | 3 ++- scripts/template-ledger-builds-test.sh | 2 +- scripts/template-ledger-run-test.sh | 15 ++++++++++-- sdk/extensionrepo/ledger_findings.go | 24 +++++++++++++------ .../.agents/skills/nicely-extension/SKILL.md | 2 +- .../nicely-extension/readiness-review.md | 4 ++-- template/AGENTS/rule-index.md.jinja | 1 + 8 files changed, 40 insertions(+), 17 deletions(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index a8d51b41..f0a623e7 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -73,7 +73,7 @@ The reviewer of a later round gives each earlier finding one state, in a line `- | `limit` | The user accepted the loss | `- **Spec**:` with the ID of an accepted row under **Limits.** | | `declined` | The author does not fix it | `- **Reason**:` with the reason of the author. Only an `advise` can be declined | -A finding of the last round can have no state yet. +A finding of the last round can have no state yet. Then it is `open`, as the state `open` says below (#423). `just finish` fails on each of these: @@ -87,8 +87,8 @@ A finding of the last round can have no state yet. - A step of round 1 that is missing, out of order, or in the commit of the step before it - No part `## First failing run` outside fenced code. Also a part without the Scenario commit, without its command, without an exit status other than 0, or without its assertion - A finding without an ID, a finding of an earlier round without a state, or a state without its parts -- A finding with the state `open`, even when `## Open lines` holds `None.` -- A finding with the state `open` whose ID no line under `## Open lines` of `spec.md` names. So `spec.md` and the ledger agree on the open work (#420) +- A finding with the state `open`, or a finding of the last round with no state, even when `## Open lines` holds `None.` +- A finding with the state `open`, or a finding of the last round with no state, whose ID no line under `## Open lines` of `spec.md` names. So `spec.md` and the ledger agree on the open work (#420, #423) - A `warn` or `block` finding with the state `declined` - A finding of an earlier committed `readiness.md` that is missing, or whose severity went down. A finding without an ID keeps its title - A committed review whose `Commit:` is the Head of no round, or a round before the last without a committed review of its Head diff --git a/scripts/template-finish-test.sh b/scripts/template-finish-test.sh index b118036a..c0df2a64 100644 --- a/scripts/template-finish-test.sh +++ b/scripts/template-finish-test.sh @@ -59,7 +59,8 @@ fct_test_finish() { fct_finish 1 fct_finish_said 'readiness.md: it names no reviewer' fct_finish_said 'readiness.md:8: the verdict is Ready, but this finding has the severity warn' - fct_readiness "${sha}" Ready "${advise}" + # An advise of this round with no state is open, so the author declines it (#423). + fct_readiness "${sha}" Ready "${advise}"$'\n- **State**: `declined`\n- **Reason**: the answer stays small' git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'independent readiness review' ledger_earlier="${sha}" diff --git a/scripts/template-ledger-builds-test.sh b/scripts/template-ledger-builds-test.sh index 54f294af..7a840157 100644 --- a/scripts/template-ledger-builds-test.sh +++ b/scripts/template-ledger-builds-test.sh @@ -56,7 +56,7 @@ fct_test_ledger_builds() { fct_finish_said 'this table names the severity block' # Finish reads a severity only in the line - **Severity**:, so the words # block and warn pass in a title, in evidence, and in prose (#419). - fct_readiness "${head}" Ready "${valid}"$'\n- **Evidence**: the code block in README.md\n\nThe CLI does not warn when the input is empty.\n\n### R3-F1 Large inputs block the terminal\n\n- **Severity**: `advise`\n\n## Notes\n\nA long line can block a pager. R3-F1 and R2-F1 name such a case.' 'agent B' + fct_readiness "${head}" Ready "${valid}"$'\n- **Evidence**: the code block in README.md\n\nThe CLI does not warn when the input is empty.\n\n### R3-F1 Large inputs block the terminal\n\n- **Severity**: `advise`\n- **State**: `declined`\n- **Reason**: a pager can block too\n\n## Notes\n\nA long line can block a pager. R3-F1 and R2-F1 name such a case.' 'agent B' fct_finish 0 # Finish reads the header in the first three lines, after a byte order mark (#416). fct_readiness "${head}" Ready "${valid}" 'agent B' diff --git a/scripts/template-ledger-run-test.sh b/scripts/template-ledger-run-test.sh index bea7d25d..6c320569 100644 --- a/scripts/template-ledger-run-test.sh +++ b/scripts/template-ledger-run-test.sh @@ -2,8 +2,9 @@ # Sourced by template-ledger-test.sh. fct_test_ledger calls fct_test_ledger_run # at the end of round 3, with its repository and its fixture variables. It # proves the record of the first failing run of round 1, and that ## Open -# lines names each open finding (#420). Markdown backticks in the fixture -# text are literal, and fct_readiness reads ledger_earlier and ledger_run. +# lines names each open finding, also one of this round with no state (#420, +# #423). Markdown backticks in the fixture text are literal, and +# fct_readiness reads ledger_earlier and ledger_run. # shellcheck disable=SC2016,SC2034,SC2154 fct_test_ledger_run() { @@ -16,6 +17,16 @@ fct_test_ledger_run() { fct_finish 1 fct_finish_said 'spec.md: ## Open lines does not name the open finding' absent git -C "${finish}" checkout -q spec.md + # A finding of this round with no state is open, beside None. too (#423). + fct_readiness "${head}" 'Needs work' "${valid}"$'\n\n### R3-F1 Long lines are cut\n\n- **Severity**: `advise`' 'agent B' + fct_finish 1 + fct_finish_said 'the finding R3-F1 of this round has no state, so it is open, even if ## Open lines holds None.' + fct_finish_said 'spec.md: ## Open lines does not name the open finding R3-F1 of readiness.md' + fct_edit "${finish}/spec.md" 'None.' $'| ID | Command | Part | Case | Code |\n|---|---|---|---|---|\n| `long-lines` | inspect | Failures | R3-F1: a line over 1 MB is cut | `RUNTIME` |' + fct_finish 1 + fct_finish_said 'the finding R3-F1 of this round has no state, so it is open' + fct_finish_said 'spec.md: ## Open lines does not name the open finding' absent + git -C "${finish}" checkout -q spec.md # Round 4 stops at a limit with R4-F1 open and named under ## Open lines. # The user accepts it, so its row moves from ## Open lines to **Limits.**, diff --git a/sdk/extensionrepo/ledger_findings.go b/sdk/extensionrepo/ledger_findings.go index 097b9035..4be735ee 100644 --- a/sdk/extensionrepo/ledger_findings.go +++ b/sdk/extensionrepo/ledger_findings.go @@ -9,7 +9,7 @@ import ( ) // stateProblems checks the state of each finding in the build of n rounds. -// A new finding of round n can have no state yet. +// A new finding of round n can have no state yet, and then it is open (#423). func stateProblems(findings []finding, n int, spec string, limits []limit) []string { var proven []string for _, l := range SpecLines(spec) { @@ -45,15 +45,13 @@ func stateProblems(findings []finding, n int, spec string, limits []limit) []str case "": if first < n { problems = append(problems, fmt.Sprintf("%sthe finding %s of round %d has no state. Fix: the reviewer of this round gives it the state open, fixed, limit, or declined", at, f.id, first)) + continue } + problems = append(problems, fmt.Sprintf("%sthe finding %s of this round has no state, so it is open, even if ## Open lines holds None. Fix: prove its spec ID in a round, or ask the user to accept it as a limit", at, f.id)) + problems = append(problems, openLineProblems(f.id, open)...) case "open": problems = append(problems, fmt.Sprintf("%sthe finding %s is open, even if ## Open lines holds None. Fix: prove its spec ID in a round, or ask the user to accept it as a limit", at, f.id)) - // spec.md and the ledger give one state of the open work (#420). - named := regexp.MustCompile(`\b` + regexp.QuoteMeta(f.id) + `\b`) - if !slices.ContainsFunc(open, named.MatchString) { - problems = append(problems, fmt.Sprintf("spec.md: ## Open lines does not name the open finding %s of readiness.md. "+ - "Fix: list it under ## Open lines by its ID, such as in the Case of its row, so spec.md and the ledger agree on the open work", f.id)) - } + problems = append(problems, openLineProblems(f.id, open)...) case "fixed": if commit, ok := ledgerCommit(f.commit); !ok || commit == "" { problems = append(problems, at+"this fixed finding names no commit of HEAD. Fix: name the commit that fixed it in a line - **Commit**: ") @@ -78,3 +76,15 @@ func stateProblems(findings []finding, n int, spec string, limits []limit) []str } return problems } + +// openLineProblems reports an open finding whose ID no line under ## Open +// lines names. So spec.md and the ledger give one state of the open work +// (#420). +func openLineProblems(id string, open []string) []string { + named := regexp.MustCompile(`\b` + regexp.QuoteMeta(id) + `\b`) + if slices.ContainsFunc(open, named.MatchString) { + return nil + } + return []string{fmt.Sprintf("spec.md: ## Open lines does not name the open finding %s of readiness.md. "+ + "Fix: list it under ## Open lines by its ID, such as in the Case of its row, so spec.md and the ledger agree on the open work", id)} +} diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index 6ee0ce1e..9f3d01b1 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -91,7 +91,7 @@ The first round is incomplete by design. The open lines start the next round. 3. Run `just acceptance` on the real input until its independent checks pass. 4. Ask an agent that did not write the round to review the changes of the round. Give it the commits of the round. 5. Ask an agent that did not write the code to run the readiness review. It records the review and the ledger in `readiness.md`. One independent pass can do both reviews. -6. Write each finding that a test can prove as an open line, which names the finding by its ID. Each finding that the ledger records as `open` keeps its line under `## Open lines`, so `spec.md` and the ledger agree. +6. Write each finding that a test can prove as an open line, which names the finding by its ID. Each finding that the ledger records as `open` keeps its line under `## Open lines`, so `spec.md` and the ledger agree. A finding of this round with no state is `open` too. 7. Run `just finish`. Stop as "Done when" in `AGENTS.md` says. `just finish` fails until the loop is done. 8. When `just finish` fails after round 5, stop, as "Round limit" says. 9. Otherwise, start the next round from the open lines and the open findings. Add its commits to the open pull request, or open a new one. Base a new pull request on the branch of the previous one when that one is not merged. diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index dc17d901..25f95822 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -29,7 +29,7 @@ Write each part of a finding as a bullet that starts with its bold name, such as - **Command**: the command that the finding affects - **Consequence**: what goes wrong for a user or an agent - **Severity**: `block` when the command breaks the contract, `warn` when the adapter works around a loss, or `advise` for a better practice -- **State**: `open`, `fixed`, `limit`, or `declined`. A finding of this round can have no state yet +- **State**: `open`, `fixed`, `limit`, or `declined`. A finding of this round can have no state yet, and then it is `open` - **Spec**: the spec ID of its line: the open line, the line that the test of the fix proves, or the accepted limit - **Commit**: for a `fixed` finding, the commit that fixed it - **Reason**: for a `declined` finding, the reason of the author @@ -65,7 +65,7 @@ Never lower the severity of a finding in a later round. A `warn` or `block` finding closes only as `fixed` or `limit`. Only an `advise` can be `declined`. A change of `spec.md` that narrows a promise to close a `warn` or `block` finding is a `limit`. Only the user can accept it. `None.` under `## Open lines` does not close a finding that the ledger records as `open`. -Each finding with the state `open` keeps a line under `## Open lines` of `spec.md` that names its ID. `just finish` fails without it. +Each finding with the state `open` keeps a line under `## Open lines` of `spec.md` that names its ID. `just finish` fails without it. A finding of this round with no state is `open` too. End the review with one verdict. The verdict follows from the findings, and `just finish` checks it: diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index 317223da..88a355cf 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -40,6 +40,7 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | The ledger names the head, the reviewers, the commits that each review covered, and the result of `just finish` of each round | [Ledger][ledger] | | Each finding keeps its ID and its highest severity, and gets a state each round; a `warn` or `block` closes only as `fixed` or `limit` | [Ledger findings][ledger-findings], [ledger history][ledger-history] | | `## Open lines` of `spec.md` names each finding that the ledger records as `open`, by its ID | [Ledger findings][ledger-findings] | +| A finding of the last round with no state is `open` | [Ledger findings][ledger-findings] | | A severity is read only in the line **Severity** of a finding; the plain words block, warn, and advise pass elsewhere | [Readiness findings][readiness] | | A finding heading stays under `## Findings`, and no other form grades | [Readiness findings][readiness] | | Finish finds a grade in code, at the end of a cell, after severity or grade, or before level | [Readiness findings][readiness] | From 066d48f59bd56f834dd534edea81f3c2909985c2 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:22:10 -0400 Subject: [PATCH 44/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20sdk/exte?= =?UTF-8?q?nsiontest:=20the=20coverage=20report=20moves=20to=20its=20own?= =?UTF-8?q?=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cleanup of Run that prints the reservations and the missing coverage moves to kit.reportCoverage in coverage_report.go, with no change of behavior, so a run of one scenario can change it. Co-Authored-By: Claude Opus 5.5 --- sdk/extensiontest/coverage_report.go | 21 +++++++++++++++++++++ sdk/extensiontest/extensiontest.go | 12 +----------- 2 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 sdk/extensiontest/coverage_report.go diff --git a/sdk/extensiontest/coverage_report.go b/sdk/extensiontest/coverage_report.go new file mode 100644 index 00000000..f61452e7 --- /dev/null +++ b/sdk/extensiontest/coverage_report.go @@ -0,0 +1,21 @@ +package extensiontest + +import ( + "testing" + + "github.com/pascalandy/nicely/sdk/extensionrepo" +) + +// reportCoverage prints each deferral of a declared code, and fails t for +// each command or code that no scenario observed. Run calls it after all +// scenarios finish. +func (k *kit) reportCoverage(t *testing.T) { + for _, reservation := range k.Reservations() { + if err := extensionrepo.Notice(reservation); err != nil { + t.Errorf("extensionrepo.Notice: %v", err) + } + } + for _, problem := range k.Missing() { + t.Error(problem) + } +} diff --git a/sdk/extensiontest/extensiontest.go b/sdk/extensiontest/extensiontest.go index b7c8288f..3c7559ac 100644 --- a/sdk/extensiontest/extensiontest.go +++ b/sdk/extensiontest/extensiontest.go @@ -53,7 +53,6 @@ import ( "testing" "github.com/pascalandy/nicely/sdk/extension" - "github.com/pascalandy/nicely/sdk/extensionrepo" "github.com/rogpeppe/go-internal/testscript" ) @@ -83,16 +82,7 @@ func Run(t *testing.T, e extension.Extension) { Cmds: examples.commands(), }) // testscript starts parallel subtests. Cleanup runs after all scenarios finish. - t.Cleanup(func() { - for _, reservation := range k.Reservations() { - if err := extensionrepo.Notice(reservation); err != nil { - t.Errorf("extensionrepo.Notice: %v", err) - } - } - for _, problem := range k.Missing() { - t.Error(problem) - } - }) + t.Cleanup(func() { k.reportCoverage(t) }) if err := k.checkInstallOrder(t.TempDir()); err != nil { t.Fatal(err) } From 08f05e714ad9c9b244e1aae87da56e46b9c8b006 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:21:01 -0400 Subject: [PATCH 45/61] =?UTF-8?q?=E2=9C=A8=20feat:=20template:=20just=20si?= =?UTF-8?q?gnoff=20runs=20the=20first=20failing=20run=20again?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A record of ## First failing run passed when its form was right, also with the command `echo invented` or with a failure that was a compile error. The record now names the recipe just scenario , of a scenario of the Scenario commit, and its assertion holds a line of that scenario. After just finish, just signoff checks out the Scenario commit in a new scratch worktree and runs that scenario alone with go test. The run must fail on the line that the record names. Signoff builds the arguments from the name of the scenario, and never runs the text of the record. just scenario runs one scenario. The test kit then skips its coverage checks, which need every scenario. Refs #422, #423 Co-Authored-By: Claude Opus 5.5 --- .../contract/extension-repository.md | 3 +- docs/north-star/contract/finish-gate.md | 17 ++- scripts/template-finish-helpers.sh | 2 +- scripts/template-ledger-run-test.sh | 25 ++++- scripts/template-signoff-test.sh | 80 +++++++++++-- sdk/extensionrepo/ledger_run.go | 105 +++++++++++++----- sdk/extensionrepo/program.go | 5 +- sdk/extensionrepo/recipes.go | 4 + sdk/extensionrepo/scenario.go | 39 +++++++ sdk/extensionrepo/signoff.go | 14 ++- sdk/extensionrepo/signoff_receipt.go | 12 +- sdk/extensionrepo/signoff_replay.go | 95 ++++++++++++++++ sdk/extensiontest/coverage_report.go | 7 +- .../.agents/skills/nicely-extension/SKILL.md | 2 +- .../nicely-extension/readiness-review.md | 6 +- template/AGENTS.md.jinja | 3 +- template/AGENTS/rule-index.md.jinja | 3 + template/justfile | 4 + 18 files changed, 365 insertions(+), 61 deletions(-) create mode 100644 sdk/extensionrepo/scenario.go create mode 100644 sdk/extensionrepo/signoff_replay.go diff --git a/docs/north-star/contract/extension-repository.md b/docs/north-star/contract/extension-repository.md index 9a4418ed..ca3689bb 100644 --- a/docs/north-star/contract/extension-repository.md +++ b/docs/north-star/contract/extension-repository.md @@ -37,7 +37,8 @@ The repository tests itself through the [test kit](test-kit.md). Its `justfile` - `just check` runs every check: `go mod tidy -diff`, `gofmt`, `go vet`, and `go test`, whose test kit installs the extension into scratch folders - `just finish` is the exit of the loop of the skill, and `just check` stays green for an incomplete round. [Finish gate](finish-gate.md) lists what it checks -- `just signoff` runs `just finish` and `just check` on the committed SHA in isolation, then signs that SHA, as [Local checks](../decisions/D016-local-ci.md) defines +- `just signoff` runs `just finish` and `just check` on the committed SHA in isolation, then signs that SHA, as [Local checks](../decisions/D016-local-ci.md) defines. After `just finish`, it runs the first failing run again, as [Finish gate](finish-gate.md#the-first-failing-run-again) says +- `just scenario ` runs the scenario `testdata/script/.txtar` alone, and the test kit skips its coverage checks - `just release [fixture]` publishes the signed SHA and proves the README install, as [Release and install](extensions.md#release-and-install) defines - `just install` builds the executable and exports its sidecars from that build. It replaces the previous install of the slug in the extensions folder of [Discovery](discovery.md), the executable last - `just uninstall` removes the install of the slug from that folder, the executable first. With nothing installed, it says so and exits 0 diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index f0a623e7..5c318021 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -56,12 +56,24 @@ Under `## Order of round 1`, a table with the columns Step and Commit names the Under `## First failing run`, four lines record the run of the first scenario of the current build at its commit, before the handler (#420): - `- **Commit**:` the Scenario commit of `## Order of round 1`, so a record of an earlier build never stands in -- `- **Command**:` the command that ran the scenario +- `- **Command**:` `just scenario `, of a scenario `testdata/script/.txtar` that the Scenario commit holds (#422) - `- **Exit**:` its exit status, which is not 0 -- `- **Assertion**:` the assertion of the scenario that failed +- `- **Assertion**:` the assertion of the scenario that failed. It holds the text of that line of the scenario The reviewer refuses a failure that is only a compile error, because it does not prove the missing behavior. +### The first failing run again + +`just signoff` runs `just finish`, then the first failing run of `readiness.md` again (#422): + +1. It reads the name of the scenario from the command `just scenario `. It never runs the text of the record as a command +2. It checks out the Scenario commit in a new scratch worktree, outside the repository +3. It runs that scenario alone with `go test`, as `just scenario` does +4. The run must fail on a line of the scenario, and the recorded assertion must hold the text of that line + +Signoff signs nothing when the run passes, or fails before any line of the scenario, such as on a compile error. +The receipt of signoff records the run under `first_run`. + Each finding under `## Findings` starts its heading with a stable ID, such as `### R2-F3`. `R2` is the round that first reported it, and the ID stays in later rounds. The reviewer of a later round gives each earlier finding one state, in a line `- **State**:`: @@ -86,6 +98,7 @@ A finding of the last round can have no state yet. Then it is `open`, as the sta - A last round whose Head is not the `Commit:` of `readiness.md` - A step of round 1 that is missing, out of order, or in the commit of the step before it - No part `## First failing run` outside fenced code. Also a part without the Scenario commit, without its command, without an exit status other than 0, or without its assertion +- A command of `## First failing run` that is not `just scenario ` of a scenario of the Scenario commit, or an assertion that holds no line of that scenario - A finding without an ID, a finding of an earlier round without a state, or a state without its parts - A finding with the state `open`, or a finding of the last round with no state, even when `## Open lines` holds `None.` - A finding with the state `open`, or a finding of the last round with no state, whose ID no line under `## Open lines` of `spec.md` names. So `spec.md` and the ledger agree on the open work (#420, #423) diff --git a/scripts/template-finish-helpers.sh b/scripts/template-finish-helpers.sh index 410dc4cf..a00aeeea 100644 --- a/scripts/template-finish-helpers.sh +++ b/scripts/template-finish-helpers.sh @@ -63,7 +63,7 @@ fct_step() { # assertion. Empty, it leaves the part out (#420). fct_ledger() { local heads first i head finish_cell handler="" run - run=$'- **Commit**: '"$(fct_step "${1}" scenario)"$'\n- **Command**: `go test ./... -run TestExtension/inspect`\n- **Exit**: 1\n- **Assertion**: `stdout \'"lines":3\'` of testdata/script/inspect.txtar found no match' + run=$'- **Commit**: '"$(fct_step "${1}" scenario)"$'\n- **Command**: `just scenario inspect`\n- **Exit**: 1\n- **Assertion**: `stdout \'"lines":3\'` of testdata/script/inspect.txtar found no match' read -r -a heads <<<"${2}" first="$(git -C "${1}" rev-list --max-parents=0 HEAD)" printf '\n## Rounds\n\n| Round | Head | Change review | Change commits | Readiness review | Finish |\n|---|---|---|---|---|---|\n' diff --git a/scripts/template-ledger-run-test.sh b/scripts/template-ledger-run-test.sh index 6c320569..9d323bf6 100644 --- a/scripts/template-ledger-run-test.sh +++ b/scripts/template-ledger-run-test.sh @@ -3,7 +3,7 @@ # at the end of round 3, with its repository and its fixture variables. It # proves the record of the first failing run of round 1, and that ## Open # lines names each open finding, also one of this round with no state (#420, -# #423). Markdown backticks in the fixture text are literal, and +# #422, #423). Markdown backticks in the fixture text are literal, and # fct_readiness reads ledger_earlier and ledger_run. # shellcheck disable=SC2016,SC2034,SC2154 @@ -59,7 +59,7 @@ fct_test_ledger_run() { ledger_run='' fct_readiness "${head}" Ready "${valid}" 'agent B' fct_finish 1 fct_finish_said 'readiness.md: it holds no part ## First failing run' - ledger_run=$'- **Command**: `go test ./...`\n- **Exit**: 0\n- **Assertion**:' fct_readiness "${head}" Ready "${valid}" 'agent B' + ledger_run=$'- **Command**: `just scenario inspect`\n- **Exit**: 0\n- **Assertion**:' fct_readiness "${head}" Ready "${valid}" 'agent B' fct_finish 1 fct_finish_said 'the first run of the scenario exited 0' fct_finish_said 'names no assertion that failed' @@ -68,14 +68,29 @@ fct_test_ledger_run() { fct_finish_said 'names no command' fct_finish_said 'the exit status "one" of the first run' # A record of another commit, such as of an earlier build, or in fenced code, records no run of this build. - ledger_run=$'- **Commit**: '"${intake}"$'\n- **Command**: `go test ./...`\n- **Exit**: 1\n- **Assertion**: `stdout 3`' fct_readiness "${head}" Ready "${valid}" 'agent B' + ledger_run=$'- **Commit**: '"${intake}"$'\n- **Command**: `just scenario inspect`\n- **Exit**: 1\n- **Assertion**: `stdout 3`' fct_readiness "${head}" Ready "${valid}" 'agent B' fct_finish 1 fct_finish_said "names the commit \"${intake}\", which is not the Scenario commit of ## Order of round 1" - ledger_run=$'```\n- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `go test ./...`\n- **Exit**: 1\n- **Assertion**: `stdout 3`\n```' fct_readiness "${head}" Ready "${valid}" 'agent B' + ledger_run=$'```\n- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `just scenario inspect`\n- **Exit**: 1\n- **Assertion**: `stdout 3`\n```' fct_readiness "${head}" Ready "${valid}" 'agent B' fct_finish 1 fct_finish_said 'names no command' # A longer fence holds a shorter one, and a value can end with a period. - ledger_run=$'````\n```\n````\n\n- **Commit**: '"$(fct_step "${finish}" scenario)"$'.\n- **Command:** `go test ./...`\n- **Exit**: 1.\n- **Assertion**: `stdout 3` found no match.' fct_readiness "${head}" Ready "${valid}" 'agent B' + ledger_run=$'````\n```\n````\n\n- **Commit**: '"$(fct_step "${finish}" scenario)"$'.\n- **Command:** `just scenario inspect`\n- **Exit**: 1.\n- **Assertion**: `stdout \'"lines":3\'` found no match.' fct_readiness "${head}" Ready "${valid}" 'agent B' fct_finish 0 + + # The command is just scenario , of a scenario of the Scenario commit, and the + # assertion holds a line of that scenario. So signoff can run it again (#422). + ledger_run=$'- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `echo invented`\n- **Exit**: 1\n- **Assertion**: `stdout \'"lines":3\'`' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said 'names the command "echo invented", which is not just scenario ' + ledger_run=$'- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `just scenario inspect; echo invented`\n- **Exit**: 1\n- **Assertion**: `stdout \'"lines":3\'`' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said 'names the command "just scenario inspect; echo invented", which is not just scenario ' + ledger_run=$'- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `just scenario missing`\n- **Exit**: 1\n- **Assertion**: `stdout \'"lines":3\'`' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said 'runs the scenario testdata/script/missing.txtar, which the Scenario commit' + ledger_run=$'- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `just scenario inspect`\n- **Exit**: 1\n- **Assertion**: the count was wrong' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said 'names the assertion "the count was wrong", which holds no line of testdata/script/inspect.txtar' git -C "${finish}" checkout -q readiness.md } diff --git a/scripts/template-signoff-test.sh b/scripts/template-signoff-test.sh index b2c86d45..e33959b6 100644 --- a/scripts/template-signoff-test.sh +++ b/scripts/template-signoff-test.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash # Sourced by template-test.sh to prove just signoff of the rendered # repository. fct_test_uninstall calls it after its install tests. +# Markdown backticks in the fixture text are literal. +# shellcheck disable=SC2016 # Run the rendered recipe with real Git and just. Only gh is a stub. fct_test_signoff() { @@ -24,25 +26,83 @@ SCRIPT fi [[ ! -e "${scratch}/signed-sha" ]] || fct_fail 'just signoff called gh signoff before just finish passed' - cp -R "${repo}" "${ready}" - fct_steps "${ready}" - for file in vision.md spec.md qa.md; do - grep -v '^Fill:' "${ready}/${file}" >"${scratch}/form.md" - mv "${scratch}/form.md" "${ready}/${file}" - done - git -C "${ready}" commit -qam 'fill the forms' - finish="${ready}" fct_readiness "$(git -C "${ready}" rev-parse HEAD)" Ready None. - git -C "${ready}" add readiness.md - git -C "${ready}" commit -qm 'readiness review' + # Signoff runs the first failing run again at its Scenario commit (#422). + # An invented command, a compile error, and a scenario that passes sign nothing. + fct_signoff_ready "${repo}" "${scratch}" "${scratch}/signoff-echo" read + fct_signoff_refuses "${scratch}" "${scratch}/signoff-echo" 'echo invented' 'names the command "echo invented", which is not just scenario ' + fct_signoff_ready "${repo}" "${scratch}" "${scratch}/signoff-compile" read 'package broken'$'\n''func {' + fct_signoff_refuses "${scratch}" "${scratch}/signoff-compile" 'just scenario first' 'before any line of the scenario, such as on a compile error' + fct_signoff_ready "${repo}" "${scratch}" "${scratch}/signoff-passes" inspect + fct_signoff_refuses "${scratch}" "${scratch}/signoff-passes" 'just scenario first' 'passes at its Scenario commit' + + fct_signoff_ready "${repo}" "${scratch}" "${ready}" read + fct_signoff_review "${ready}" 'just scenario first' sha="$(git -C "${ready}" rev-parse HEAD)" if ! fct_signoff "${ready}" "${scratch}"; then cat "${scratch}/signoff.out" >&2 fct_fail 'rendered just signoff failed' fi fct_has "${scratch}/signoff.out" "Signed off on ${sha}" + grep -qF 'signoff: the first scenario fails again at' "${scratch}/signoff.out" || fct_fail 'just signoff did not run the first failing run again' fct_has "${scratch}/signed-sha" "${sha}" } +# fct_signoff_ready copies the rendered repository $1 to $3, with the steps of +# round 1 and filled forms, in the scratch folder $2. Its first scenario +# expects a message that ends with "can $4". "read" fails until the handler +# changes the message, and "inspect" passes at once. A Go file $5 in the +# Scenario commit, removed by the handler, breaks the build of that commit. +fct_signoff_ready() { + local ready="${3}" file refuses expected + cp -R "${1}" "${ready}" + git -C "${ready}" commit -q --allow-empty -m 'step: intake' + git -C "${ready}" commit -q --allow-empty -m 'step: spec' + refuses="$(grep -m 1 '^refuses USAGE_INVALID ncly .* missing.md --json$' "${ready}/testdata/script/inspect.txtar")" + expected="$(grep -m 1 "^stderr 'missing.md is no file that .* can inspect\[.\]'$" "${ready}/testdata/script/inspect.txtar")" + printf '# A missing file names the cause in plain words.\n# spec: bad-input\n%s\n%s\n' "${refuses}" "${expected/can inspect/can ${4}}" >"${ready}/testdata/script/first.txtar" + [[ -z "${5:-}" ]] || printf '%s\n' "${5}" >"${ready}/broken.go" + git -C "${ready}" add -A + git -C "${ready}" commit -qm 'step: scenario' + [[ -z "${5:-}" ]] || git -C "${ready}" rm -q broken.go + if [[ "${4}" != inspect ]]; then + for file in locales/en.toml testdata/script/inspect.txtar; do + sed "s/ can inspect\([.[]\)/ can ${4}\1/" "${ready}/${file}" >"${2}/form.md" + mv "${2}/form.md" "${ready}/${file}" + done + fi + git -C "${ready}" commit -q --allow-empty -am 'handler' + for file in vision.md spec.md qa.md; do + grep -v '^Fill:' "${ready}/${file}" >"${2}/form.md" + mv "${2}/form.md" "${ready}/${file}" + done + git -C "${ready}" commit -qam 'fill the forms' +} + +# fct_signoff_review commits the readiness review of the head of $1, whose +# first failing run names the command $2. +fct_signoff_review() { + local expected + expected="$(grep -m 1 '^stderr ' "${1}/testdata/script/first.txtar")" + finish="${1}" ledger_run=$'- **Commit**: '"$(fct_step "${1}" scenario)"$'\n- **Command**: `'"${2}"$'`\n- **Exit**: 1\n- **Assertion**: `'"${expected}"'` found no match' \ + fct_readiness "$(git -C "${1}" rev-parse HEAD)" Ready None. + git -C "${1}" add readiness.md + git -C "${1}" commit -qm 'readiness review' +} + +# fct_signoff_refuses requires that just signoff of the repository $2, whose +# first failing run names the command $3, fails with the text $4 and signs +# nothing. +fct_signoff_refuses() { + local status=0 + fct_signoff_review "${2}" "${3}" + fct_signoff "${2}" "${1}" || status=$? + if [[ "${status}" == 0 ]] || ! grep -qF -- "${4}" "${1}/signoff.out"; then + cat "${1}/signoff.out" >&2 + fct_fail "just signoff with the first run \"${3}\" must fail with \"${4}\"" + fi + [[ ! -e "${1}/signed-sha" ]] || fct_fail "just signoff signed a commit whose first run is \"${3}\"" +} + # fct_signoff runs just signoff in the repository $1, with the stub gh of the scratch folder $2. fct_signoff() { (cd "${1}" && PATH="${2}/signoff-bin:${PATH}" XDG_CACHE_HOME="${2}/signoff-cache" \ diff --git a/sdk/extensionrepo/ledger_run.go b/sdk/extensionrepo/ledger_run.go index 700b872f..2f9d07c6 100644 --- a/sdk/extensionrepo/ledger_run.go +++ b/sdk/extensionrepo/ledger_run.go @@ -2,6 +2,7 @@ package extensionrepo import ( "fmt" + "os/exec" "regexp" "strconv" "strings" @@ -14,12 +15,82 @@ var firstRun = regexp.MustCompile(`^\s*[-*+]\s+\*\*(Commit|Command|Exit|Assertio // firstRunProblems checks the record of the first failing run of the current // build. It needs the Scenario commit of ## Order of round 1, so a record of -// an earlier build never stands in. It also needs the command, an exit status -// that is not 0, and the assertion that failed. The reviewer judges whether -// that assertion proves the missing behavior, so a compile error is no proof. +// an earlier build never stands in. It also needs the command just scenario +// of a scenario of that commit, an exit status that is not 0, and the +// assertion that failed, which holds a line of that scenario. just signoff +// runs the scenario again, and the reviewer judges whether the assertion +// proves the missing behavior (#422). func firstRunProblems(readiness string, order map[string]round) []string { - const fix = "Fix: run the first scenario at its commit, before the handler, and record that run under ## First failing run " + + const fix = "Fix: run the first scenario at its commit with just scenario , before the handler, and record that run under ## First failing run " + "in the lines - **Commit**:, - **Command**:, - **Exit**:, and - **Assertion**:" + parts, found := firstRunRecord(readiness) + if !found { + return []string{"readiness.md: it holds no part ## First failing run, so no record shows that the first scenario of round 1 failed before the handler. " + fix} + } + var problems []string + scenario, recorded := order["Scenario"], "" + if len(scenario.cells) == 2 { + recorded, _ = ledgerCommit(scenario.cells[1]) + } + commit, _ := ledgerCommit(parts["Commit"]) + if commit == "" || commit != recorded { + problems = append(problems, fmt.Sprintf("readiness.md: ## First failing run names the commit %q, which is not the Scenario commit of ## Order of round 1. %s", parts["Commit"], fix)) + } + if parts["Command"] == "" { + problems = append(problems, "readiness.md: ## First failing run names no command. "+fix) + } else if name := scenarioRecipe.FindStringSubmatch(parts["Command"]); name == nil { + problems = append(problems, fmt.Sprintf("readiness.md: ## First failing run names the command %q, which is not just scenario . "+ + "just signoff runs only that recipe again, never the text of the record. %s", parts["Command"], fix)) + } else if commit != "" { + problems = append(problems, scenarioProblems(commit, name[1], parts["Assertion"], fix)...) + } + switch exit, err := strconv.Atoi(parts["Exit"]); { + case err != nil: + problems = append(problems, fmt.Sprintf("readiness.md: the exit status %q of the first run is no number. %s", parts["Exit"], fix)) + case exit == 0: + problems = append(problems, "readiness.md: the first run of the scenario exited 0, so it did not fail. "+fix) + } + if strings.TrimSpace(strings.TrimSuffix(parts["Assertion"], ".")) == "" { + problems = append(problems, "readiness.md: ## First failing run names no assertion that failed. "+ + "A compile error is no failed assertion, because it proves no missing behavior. "+fix) + } + return problems +} + +// scenarioProblems checks that the scenario name exists at commit, and that +// the assertion holds one of its lines. +func scenarioProblems(commit, name, assertion, fix string) []string { + file := "testdata/script/" + name + ".txtar" + script, err := exec.Command("git", "show", commit+":"+file).Output() + if err != nil { + return []string{fmt.Sprintf("readiness.md: ## First failing run runs the scenario %s, which the Scenario commit %s does not hold. %s", file, commit, fix)} + } + if strings.TrimSpace(assertion) == "" || scenarioLine(string(script), assertion) == 0 { + return []string{fmt.Sprintf("readiness.md: ## First failing run names the assertion %q, which holds no line of %s at %s. "+ + "Fix: copy the line of the scenario that failed, as the output of the run names it", assertion, file, commit)} + } + return nil +} + +// scenarioLine returns the number of the first line of the script of a +// scenario, before its files, that text holds. It skips blank lines and +// comments. It returns 0 when text holds no such line. +func scenarioLine(script, text string) int { + for i, line := range strings.Split(script, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "-- ") && strings.HasSuffix(line, " --") { + break + } + if line != "" && !strings.HasPrefix(line, "#") && strings.Contains(text, line) { + return i + 1 + } + } + return 0 +} + +// firstRunRecord reads the lines of ## First failing run, outside fenced +// code. It also says whether readiness.md holds that part. +func firstRunRecord(readiness string) (map[string]string, bool) { parts, at, found, fence := map[string]string{}, 0, false, "" for i, text := range strings.Split(readiness, "\n") { // A fence closes with its own run of marks, alone on its line, as in readinessFindings. @@ -45,29 +116,5 @@ func firstRunProblems(readiness string, order map[string]round) []string { parts[m[1]] = strings.Trim(strings.TrimSuffix(m[2], "."), "`") } } - if !found { - return []string{"readiness.md: it holds no part ## First failing run, so no record shows that the first scenario of round 1 failed before the handler. " + fix} - } - var problems []string - scenario, recorded := order["Scenario"], "" - if len(scenario.cells) == 2 { - recorded, _ = ledgerCommit(scenario.cells[1]) - } - if commit, _ := ledgerCommit(parts["Commit"]); commit == "" || commit != recorded { - problems = append(problems, fmt.Sprintf("readiness.md: ## First failing run names the commit %q, which is not the Scenario commit of ## Order of round 1. %s", parts["Commit"], fix)) - } - if parts["Command"] == "" { - problems = append(problems, "readiness.md: ## First failing run names no command. "+fix) - } - switch exit, err := strconv.Atoi(parts["Exit"]); { - case err != nil: - problems = append(problems, fmt.Sprintf("readiness.md: the exit status %q of the first run is no number. %s", parts["Exit"], fix)) - case exit == 0: - problems = append(problems, "readiness.md: the first run of the scenario exited 0, so it did not fail. "+fix) - } - if strings.TrimSpace(strings.TrimSuffix(parts["Assertion"], ".")) == "" { - problems = append(problems, "readiness.md: ## First failing run names no assertion that failed. "+ - "A compile error is no failed assertion, because it proves no missing behavior. "+fix) - } - return problems + return parts, found } diff --git a/sdk/extensionrepo/program.go b/sdk/extensionrepo/program.go index 78fe9468..a38945a1 100644 --- a/sdk/extensionrepo/program.go +++ b/sdk/extensionrepo/program.go @@ -43,6 +43,9 @@ func run(e extension.Extension, args []string) int { if len(args) == 1 && args[0] == "finish" { return finishRepository() } + if len(args) == 2 && args[0] == "scenario" { + return runScenario(args[1]) + } if len(args) == 1 && args[0] == "contract" { return report("contract", printContractDirectory()) } @@ -50,7 +53,7 @@ func run(e extension.Extension, args []string) int { return report("export", extension.Export(e, args[1])) } if len(args) != 1 || (args[0] != "install" && args[0] != "uninstall") { - fmt.Fprintln(os.Stderr, "usage: export install | uninstall | signoff | release [fixture] | check | finish | contract | export | test [go test arguments] | update vX.Y.Z /absolute/repository [--continue]") + fmt.Fprintln(os.Stderr, "usage: export install | uninstall | signoff | release [fixture] | check | finish | scenario | contract | export | test [go test arguments] | update vX.Y.Z /absolute/repository [--continue]") return 2 } action := args[0] diff --git a/sdk/extensionrepo/recipes.go b/sdk/extensionrepo/recipes.go index 03c681a8..88f388ad 100644 --- a/sdk/extensionrepo/recipes.go +++ b/sdk/extensionrepo/recipes.go @@ -25,6 +25,10 @@ func checkRepository() int { return report("check", err) } fmt.Fprintln(os.Stderr, "go test -count=1 ./...") + // Every scenario runs, so the coverage checks run too. + if err := os.Unsetenv(ScenarioEnv); err != nil { + return report("check", err) + } return Test(nil) } diff --git a/sdk/extensionrepo/scenario.go b/sdk/extensionrepo/scenario.go new file mode 100644 index 00000000..8d4854d5 --- /dev/null +++ b/sdk/extensionrepo/scenario.go @@ -0,0 +1,39 @@ +package extensionrepo + +import ( + "fmt" + "os" + "path/filepath" + "regexp" +) + +// ScenarioEnv names the scenario that a run of go test runs alone, for just +// scenario and for the repeat of signoff. The test kit then skips its +// coverage checks, which need every scenario. +const ScenarioEnv = "NCLY_TEST_SCENARIO" + +// scenarioRecipe is the one command that ## First failing run can record: +// just scenario . Finish and signoff read the name of the scenario from +// it, and never run the text of the record (#422). +var scenarioRecipe = regexp.MustCompile(`^just scenario ([A-Za-z0-9][A-Za-z0-9_-]*)$`) + +// scenarioArgs gives the go test arguments that run the scenario +// testdata/script/.txtar. The test kit runs each scenario as a subtest +// of TestExtension, as test-kit.md in the contract says. +func scenarioArgs(name string) []string { + return []string{"-run", "^TestExtension$/^" + name + "$", "./..."} +} + +// runScenario runs just scenario , through Test. +func runScenario(name string) int { + if !scenarioRecipe.MatchString("just scenario " + name) { + return report("scenario", fmt.Errorf("%q is no name of a scenario. Fix: give the name of a file of testdata/script/ without .txtar, such as inspect", name)) + } + if _, err := os.Stat(filepath.Join("testdata", "script", name+".txtar")); err != nil { + return report("scenario", fmt.Errorf("testdata/script/%s.txtar: %w. Fix: give the name of a scenario of this commit", name, err)) + } + if err := os.Setenv(ScenarioEnv, name); err != nil { + return report("scenario", err) + } + return Test(scenarioArgs(name)) +} diff --git a/sdk/extensionrepo/signoff.go b/sdk/extensionrepo/signoff.go index 774d1f43..9ca3ecdd 100644 --- a/sdk/extensionrepo/signoff.go +++ b/sdk/extensionrepo/signoff.go @@ -84,6 +84,18 @@ func signoff(recipes []string) error { if err != nil { return fmt.Errorf("%s of %s failed: %w", recipe, sha, err) } + if recipe == "finish" { + // The ledger that finish accepted names the first failing run, which must fail again (#422). + replay, err := replayFirstRun(ctx, env, root, worktree, run, log) + receipt.FirstRun = replay + if writeErr := receipt.write(run); writeErr != nil { + return writeErr + } + if err != nil { + return fmt.Errorf("the first failing run of %s: %w", sha, err) + } + fmt.Fprintf(os.Stderr, "signoff: the first scenario fails again at %s, on %q\n", replay.Commit, replay.Failed) + } } if err := signoffUnchanged(ctx, env, worktree, sha); err != nil { return err @@ -133,7 +145,7 @@ func signoffEnv() []string { continue } switch name { - case "GOFLAGS", "GOWORK": + case "GOFLAGS", "GOWORK", ScenarioEnv: continue } env = append(env, entry) diff --git a/sdk/extensionrepo/signoff_receipt.go b/sdk/extensionrepo/signoff_receipt.go index 00dc668c..96b8d43d 100644 --- a/sdk/extensionrepo/signoff_receipt.go +++ b/sdk/extensionrepo/signoff_receipt.go @@ -17,11 +17,13 @@ type signoffCheck struct { } type signoffReceipt struct { - SHA string `json:"sha"` - OS string `json:"os"` - Tools map[string]string `json:"tools"` - Checks []signoffCheck `json:"checks"` - Signoff *signoffCheck `json:"signoff"` + SHA string `json:"sha"` + OS string `json:"os"` + Tools map[string]string `json:"tools"` + Checks []signoffCheck `json:"checks"` + // FirstRun is the repeat of the first failing run, after just finish. + FirstRun *signoffReplay `json:"first_run,omitempty"` + Signoff *signoffCheck `json:"signoff"` } func (r signoffReceipt) write(run string) error { diff --git a/sdk/extensionrepo/signoff_replay.go b/sdk/extensionrepo/signoff_replay.go new file mode 100644 index 00000000..6ecc932d --- /dev/null +++ b/sdk/extensionrepo/signoff_replay.go @@ -0,0 +1,95 @@ +package extensionrepo + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" +) + +// signoffReplay records the run of the first scenario that signoff repeats. +type signoffReplay struct { + Commit string `json:"commit"` + Command []string `json:"command"` + Exit int `json:"exit"` + // Failed is the line of the scenario that failed in the repeat. + Failed string `json:"failed"` +} + +// replayFirstRun runs the scenario of ## First failing run of the readiness.md +// of worktree again, at its Scenario commit, in a new worktree under run. It +// builds the go test arguments from the name of the scenario, so the text of +// the record never runs (#422). The run must fail on a line of the scenario +// that the recorded assertion holds. So an invented command, or a compile +// error, signs nothing. +func replayFirstRun(ctx context.Context, env []string, root, worktree, run string, log io.Writer) (*signoffReplay, error) { + const fix = "Fix: run the first scenario at its Scenario commit with just scenario , and record that run under ## First failing run" + data, err := os.ReadFile(filepath.Join(worktree, "readiness.md")) + if err != nil { + return nil, err + } + readiness := strings.TrimPrefix(string(data), "\ufeff") + parts, _ := firstRunRecord(readiness) + _, order, _ := ledgerTables(readiness) + name := scenarioRecipe.FindStringSubmatch(parts["Command"]) + scenario := order["Scenario"] + if name == nil || len(scenario.cells) != 2 { + return nil, fmt.Errorf("readiness.md records no command just scenario and no Scenario commit to run again. %s", fix) + } + commit, err := signoffOutput(ctx, env, worktree, "git", "rev-parse", "--verify", strings.Trim(scenario.cells[1], "`")+"^{commit}") + if err != nil { + return nil, err + } + dir := filepath.Join(run, "first-run") + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if output, err := signoffCommand(cleanupCtx, env, root, "git", "worktree", "remove", "--force", dir).CombinedOutput(); err != nil { + fmt.Fprintf(os.Stderr, "signoff: worktree cleanup: %v: %s\n", err, output) + } + }() + if _, err := signoffOutput(ctx, env, root, "git", "worktree", "add", "--quiet", "--detach", dir, commit); err != nil { + return nil, err + } + args := append([]string{"go", "test", "-count=1"}, scenarioArgs(name[1])...) + if _, err := fmt.Fprintf(log, "$ %s # at the Scenario commit %s\n", strings.Join(args, " "), commit); err != nil { + return nil, err + } + var output bytes.Buffer + cmd := signoffCommand(ctx, append(env, ScenarioEnv+"="+name[1]), dir, args[0], args[1:]...) + cmd.Stdout, cmd.Stderr = io.MultiWriter(&output, log), io.MultiWriter(&output, log) + replay := &signoffReplay{Commit: commit, Command: args, Exit: signoffExit(cmd.Run())} + if err := ctx.Err(); err != nil { + return replay, err + } + file := "testdata/script/" + name[1] + ".txtar" + if replay.Exit == 0 { + return replay, fmt.Errorf("the first scenario %s passes at its Scenario commit %s, so it never failed before the handler. %s", file, commit, fix) + } + failed := regexp.MustCompile(`(?m)^\s*FAIL: .*testdata/script/` + regexp.QuoteMeta(name[1]) + `\.txtar:([0-9]+): `).FindStringSubmatch(output.String()) + if failed == nil { + return replay, fmt.Errorf("the first scenario %s fails at %s before any line of the scenario, such as on a compile error, which proves no missing behavior. "+ + "The log is %s. %s", file, commit, filepath.Join(run, "check.log"), fix) + } + script, err := os.ReadFile(filepath.Join(dir, file)) + if err != nil { + return replay, err + } + number, _ := strconv.Atoi(failed[1]) + lines := strings.Split(string(script), "\n") + if number < 1 || number > len(lines) { + return replay, fmt.Errorf("the first scenario %s fails at %s on line %d, which it does not hold. %s", file, commit, number, fix) + } + replay.Failed = strings.TrimSpace(lines[number-1]) + if replay.Failed == "" || !strings.Contains(parts["Assertion"], replay.Failed) { + return replay, fmt.Errorf("the first scenario %s fails at %s on line %d, %q, and ## First failing run names the assertion %q. "+ + "Fix: record the assertion that failed, as the run names it", file, commit, number, replay.Failed, parts["Assertion"]) + } + return replay, nil +} diff --git a/sdk/extensiontest/coverage_report.go b/sdk/extensiontest/coverage_report.go index f61452e7..e9a6ce74 100644 --- a/sdk/extensiontest/coverage_report.go +++ b/sdk/extensiontest/coverage_report.go @@ -1,6 +1,7 @@ package extensiontest import ( + "os" "testing" "github.com/pascalandy/nicely/sdk/extensionrepo" @@ -8,8 +9,12 @@ import ( // reportCoverage prints each deferral of a declared code, and fails t for // each command or code that no scenario observed. Run calls it after all -// scenarios finish. +// scenarios finish. A run of one scenario, such as just scenario, skips it, +// because coverage needs every scenario (#422). func (k *kit) reportCoverage(t *testing.T) { + if os.Getenv(extensionrepo.ScenarioEnv) != "" { + return + } for _, reservation := range k.Reservations() { if err := extensionrepo.Notice(reservation); err != nil { t.Errorf("extensionrepo.Notice: %v", err) diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index 9f3d01b1..923a45f4 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -65,7 +65,7 @@ Do these steps in this order. Commit each of the first four steps on its own: 1. Record the intake facts under `## Intake` in `vision.md`, as "Evidence of the intake" in `intake.md` says. The facts include the effects, the exit codes, the resource use, and the known failures. A fact that no source gives stays an open line of `spec.md`. 2. Write every line of the command in `spec.md`, from the generic cases of the contract and the intake. Each line that has no test stays `open`, under `## Open lines`. -3. Write the first scenario at the public command boundary, with its claim `# spec: `. Commit it while it fails. Run it at that commit, and keep the command, its exit status, and the assertion that failed. A compile error is no such failure. When the scenario cannot run without the command, commit a handler that does no work in this step. Then the run reaches the assertion. +3. Write the first scenario at the public command boundary, with its claim `# spec: `. Commit it while it fails. Run it at that commit with `just scenario `. Keep that command, its exit status, and the line of the scenario that failed. A compile error is no such failure. `just signoff` runs it again. When the scenario cannot run without the command, commit a handler that does no work in this step. Then the run reaches the assertion. 4. Change the handler until that scenario passes, with "Change a command" in `AGENTS.md`. 5. When no line stays open, write `None.` under `## Open lines`, with a sentence that cites the generic cases. 6. Replace every `Fill:` line of `vision.md`, `spec.md`, and `qa.md`. diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 25f95822..8274eef0 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -86,7 +86,7 @@ A loss that you report but that the user did not accept is a `warn` finding, not In round 1, review the order of the work, not only the code: - The intake commit comes before the spec commit. The intake gives, for each fact, its source, its version, and the part that the author read, and the result of each experiment apart. A link alone is not evidence of a read -- The record of the first failing run gives the command, its exit status, and the assertion that failed at the scenario commit. Run the command again at that commit when in doubt. Refuse a failure that is only a compile error, because it does not prove the missing behavior +- The record of the first failing run gives the command, its exit status, and the assertion that failed at the scenario commit. The command is `just scenario `. Run it again at that commit when in doubt, and `just signoff` runs it again too. Refuse a failure that is only a compile error, because it does not prove the missing behavior - Each case of the Output group of `generic-cases.md` that applies has a test file that the tests make, and an independent check. A case name alone is not a test - Test families of cases, such as all the declarations that change the meaning, not one counterexample @@ -114,9 +114,9 @@ After the findings, keep the ledger of the build. Add one row for this round, an ## First failing run - **Commit**: -- **Command**: +- **Command**: just scenario - **Exit**: -- **Assertion**: +- **Assertion**: ``` Change commits names the commits that the change review covered. The readiness review covers Head. diff --git a/template/AGENTS.md.jinja b/template/AGENTS.md.jinja index a71d141e..7f6fe442 100644 --- a/template/AGENTS.md.jinja +++ b/template/AGENTS.md.jinja @@ -60,7 +60,8 @@ After Copier creates the repository, run `go mod tidy` once, which writes `go.su - `just check`: every check. It runs `go mod tidy -diff`, `gofmt`, `go vet`, and `go test`, whose test kit installs the extension into scratch folders - `just finish`: the exit of the loop. `finish-gate.md` in the pinned contract lists what it checks -- `just signoff`: runs `just finish` and `just check` on the committed SHA in a detached scratch worktree. Then it writes a receipt outside the repository, and signs that exact SHA +- `just scenario `: runs the scenario `testdata/script/.txtar`, such as the first scenario of round 1 at its commit +- `just signoff`: runs `just finish` and `just check` on the committed SHA in a detached scratch worktree. It also runs the first failing run of `readiness.md` again, at its commit. Then it writes a receipt outside the repository, and signs that exact SHA - `just install`: installs the extension into Nicely's extensions folder, in place of its previous install - `just uninstall`: removes that install, and keeps its keys and grants - `just acceptance [fixture]`: runs `acceptance.sh`, which runs the command of `## Acceptance` as an agent would, in scratch folders. It works on a copy of the fixture, or on an input that it makes diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index 88a355cf..7ea5c186 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -23,6 +23,8 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | Round 1 records the intake, then the spec, then a failing scenario, then the handler | [Finish gate][finish] checks their commits in the [ledger][ledger] | | The intake gives, for each fact, its source, its version, and the part read, with each experiment apart; a link alone is not evidence | Prose only | | Round 1 records the Scenario commit, the command, the exit status, and the failed assertion of the first scenario run | The [ledger][ledger-run] checks the record | +| The command of the first run is `just scenario `, and its assertion holds a line of that scenario | The [ledger][ledger-run] checks the record | +| `just signoff` runs the first scenario again at its commit, and never runs the text of the record | [Signoff replay][signoff-replay] | | A compile error is no failure of the first scenario; a scenario that cannot run without the command commits a handler that does no work | Prose only | | Read the source first, then run experiments only to confirm what you read | Prose only | | Each case of the Output group that applies has a test file that the tests make, checked by an independent reader | Prose only | @@ -69,5 +71,6 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". [finish-verdict]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/readiness_verdict.go [readiness]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/readiness.go [signoff]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/signoff.go +[signoff-replay]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/signoff_replay.go [release]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/release.go [template-test]: https://github.com/pascalandy/nicely/blob/main/scripts/template-test.sh diff --git a/template/justfile b/template/justfile index 8235fd55..4f110a59 100644 --- a/template/justfile +++ b/template/justfile @@ -15,6 +15,10 @@ check: finish: go run github.com/pascalandy/nicely/sdk/extensionrepo/cmd/extensionrepo finish +# Run one scenario of testdata/script/ by its name, such as just scenario inspect +scenario name: + go run github.com/pascalandy/nicely/sdk/extensionrepo/cmd/extensionrepo scenario {{ quote(name) }} + # Check the committed SHA in a detached scratch worktree, then sign that exact SHA signoff: go run ./internal/export signoff From 1944800bbb2cdeb0040a4058f25975b553592cd7 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:26:53 -0400 Subject: [PATCH 46/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20template?= =?UTF-8?q?:=20the=20proof=20of=20a=20command=20that=20replaces=20a=20file?= =?UTF-8?q?=20moves=20to=20its=20own=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The section "Commands that replace a file" of the nicely-extension skill moves to replace-a-file.md beside it, and the skill links it. Nothing changes in the rules, so the next rules of the rounds fit in SKILL.md. Co-Authored-By: Claude Opus 5.5 --- template/.agents/skills/nicely-extension/SKILL.md | 12 +----------- .../skills/nicely-extension/replace-a-file.md | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 template/.agents/skills/nicely-extension/replace-a-file.md diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index 923a45f4..61c8f2aa 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -125,17 +125,7 @@ The release receipt and the human items of `qa.md` come after the merge, as "Rel ### Commands that replace a file -For a command that replaces a user file, prove these invariants with the commands of `scenario-commands.md` in the contract: - -- The input stays unchanged: `protect` the input before each `refuses`, `signal`, and `appears` call -- A failure leaves no partial output and keeps the old output: `protect` the output path, absent or old, before each of these calls -- The command never writes through a link: give an output that is a link to the input, `protect` both, and refuse the call with `refuses` -- Without `--force`, the command never replaces a file that appears during the run: `appears` creates that file, and the call keeps it - -A `protect` before a `signal` or an `appears` call covers that one call. -For `appears`, the stub of the wrapped program writes the marker, then waits until the file exists. - -For a command that replays paid work, run `create-formal-verification-skill`. +For a command that replaces a user file, read [replace-a-file.md](replace-a-file.md). It lists the invariants to prove, and the commands of the test kit that prove them. ## Readiness review diff --git a/template/.agents/skills/nicely-extension/replace-a-file.md b/template/.agents/skills/nicely-extension/replace-a-file.md new file mode 100644 index 00000000..f784d36c --- /dev/null +++ b/template/.agents/skills/nicely-extension/replace-a-file.md @@ -0,0 +1,15 @@ +# Commands that replace a file + +This file says how to prove a command that replaces a user file. + +For a command that replaces a user file, prove these invariants with the commands of `scenario-commands.md` in the contract: + +- The input stays unchanged: `protect` the input before each `refuses`, `signal`, and `appears` call +- A failure leaves no partial output and keeps the old output: `protect` the output path, absent or old, before each of these calls +- The command never writes through a link: give an output that is a link to the input, `protect` both, and refuse the call with `refuses` +- Without `--force`, the command never replaces a file that appears during the run: `appears` creates that file, and the call keeps it + +A `protect` before a `signal` or an `appears` call covers that one call. +For `appears`, the stub of the wrapped program writes the marker, then waits until the file exists. + +For a command that replays paid work, run `create-formal-verification-skill`. From 51b7c04bfc0f89469306326a8d1f34e0333467e9 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:27:27 -0400 Subject: [PATCH 47/61] =?UTF-8?q?=E2=9C=A8=20feat:=20template:=20spec.md?= =?UTF-8?q?=20lists=20one=20case=20matrix=20before=20the=20first=20review?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each build found one variant of a family of cases at a time, and each narrow fix let the next review find a sibling. spec.md now holds a table under ## Case matrix: one row for each family of cases, from the generic cases and from the intake, with the spec IDs of the tests that pass and of the tests that refuse. just finish fails when a family names no test, when a family names no input that passes, or when a line of Failures or Acceptance is in no family. The skill asks for a pair of tests for each refusal and each branch that classifies an input, the first reviewer checks and runs the matrix before any repair, and a repair tests its whole family. Refs #423 Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 2 + docs/north-star/contract/generic-cases.md | 2 + scripts/template-finish-matrix-test.sh | 30 +++++++ scripts/template-finish-test.sh | 3 + sdk/extensionrepo/finish.go | 3 + sdk/extensionrepo/spec_matrix.go | 86 +++++++++++++++++++ .../.agents/skills/nicely-extension/SKILL.md | 16 +++- .../nicely-extension/readiness-review.md | 9 +- template/AGENTS/rule-index.md.jinja | 5 ++ template/spec.md.jinja | 18 ++++ 10 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 scripts/template-finish-matrix-test.sh create mode 100644 sdk/extensionrepo/spec_matrix.go diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index 5c318021..d27d0763 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -9,6 +9,8 @@ This part says what `just finish` checks in an extension repository, as [Extensi - A line of `vision.md`, `spec.md`, or `qa.md` that starts with `Fill:` after its indent, also in fenced code - A line under `## Open lines` - A line under **Limits.** that is no table row with a spec ID, the state `limit`, and the user's acceptance +- No table under `## Case matrix` of `spec.md` with the columns Family, Source, Accepted, and Refused (#423). Also a family that names no spec ID in Accepted, or a Refused cell with no spec ID and not `None` +- A spec ID of `## Case matrix` that is no line of Failures or Acceptance and no accepted limit, so no test proves it. Also a line of Failures or Acceptance that no family names - A `readiness.md` whose first three lines hold no `Commit: `, no `Reviewer: `, or no verdict `Ready` or `Ready with stated limits`. A byte order mark before them is ignored - A reviewed commit that differs from the working tree outside `readiness.md`, and outside the part **Limits.** of `spec.md`. So the user accepts a limit after a review, with no new round - A finding of `readiness.md` without its own `###` heading under `## Findings`, or without one severity: `block`, `warn`, or `advise` diff --git a/docs/north-star/contract/generic-cases.md b/docs/north-star/contract/generic-cases.md index 707e394c..09b92bbd 100644 --- a/docs/north-star/contract/generic-cases.md +++ b/docs/north-star/contract/generic-cases.md @@ -5,6 +5,8 @@ The spec of such a command starts its lines of Failures and Acceptance from this Then it adds the cases of the program that the command wraps. Each case that can occur becomes one line, with its spec ID, as [Spec IDs](spec-ids.md) defines. +Before the first review, `## Case matrix` of the spec gives a row to each such case and to each family of the intake (#423). +Each refusal in it has a pair of tests: one input passes, and one input is refused. The line states the outcome of the case, such as a refusal with its code. The safe outcome below is the default. Another outcome is a limit, and only the user can accept it. A case that cannot occur for the command needs no line. diff --git a/scripts/template-finish-matrix-test.sh b/scripts/template-finish-matrix-test.sh new file mode 100644 index 00000000..4c5a9d3a --- /dev/null +++ b/scripts/template-finish-matrix-test.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Sourced by template-finish-test.sh. fct_test_finish calls +# fct_test_finish_matrix on a repository that passes just finish. It proves +# that finish fails when a family of ## Case matrix in spec.md has no test, +# when a refusal has no pair, and when a line of Failures or Acceptance is in +# no family (#423). Markdown backticks in the fixture text are literal. +# shellcheck disable=SC2016,SC2154 + +fct_test_finish_matrix() { + local row='| The input holds text | Input: Of the wrong type | `lines` | `not-text` |' + fct_edit "${finish}/spec.md" "${row}" '| The input holds text | Input: Of the wrong type | `lines` | None |' + fct_finish 1 + fct_finish_said 'the line `not-text` is in no family of ## Case matrix' + fct_edit "${finish}/spec.md" '| The input holds text | Input: Of the wrong type | `lines` | None |' '| The input holds text | Input: Of the wrong type | None | `not-text` |' + fct_finish 1 + fct_finish_said 'the family "The input holds text" names no spec ID of an input that passes' + fct_edit "${finish}/spec.md" '| The input holds text | Input: Of the wrong type | None | `not-text` |' '| The input holds text | Input: Of the wrong type | `lines`, `utf-16` | `not-text` |' + fct_finish 1 + fct_finish_said 'the family "The input holds text" names `utf-16`, which is no line of Failures or Acceptance and no accepted limit' + fct_edit "${finish}/spec.md" '| The input holds text | Input: Of the wrong type | `lines`, `utf-16` | `not-text` |' '| The input holds text | Input: Of the wrong type | `lines` |' + fct_finish 1 + fct_finish_said 'this row of ## Case matrix has no cell for each column' + git -C "${finish}" checkout -q spec.md + awk '/^## Case matrix$/ { skip = 1; next } /^## / { skip = 0 } !skip' "${finish}/spec.md" >"${scratch}/spec.md" + mv "${scratch}/spec.md" "${finish}/spec.md" + fct_finish 1 + fct_finish_said 'spec.md: it holds no table under ## Case matrix' + git -C "${finish}" checkout -q spec.md + fct_finish 0 +} diff --git a/scripts/template-finish-test.sh b/scripts/template-finish-test.sh index c0df2a64..51b4fa5f 100644 --- a/scripts/template-finish-test.sh +++ b/scripts/template-finish-test.sh @@ -9,6 +9,8 @@ source "${BASH_SOURCE[0]%/*}/template-finish-helpers.sh" # shellcheck source=scripts/template-finish-forms-test.sh source "${BASH_SOURCE[0]%/*}/template-finish-forms-test.sh" +# shellcheck source=scripts/template-finish-matrix-test.sh +source "${BASH_SOURCE[0]%/*}/template-finish-matrix-test.sh" fct_test_finish() { local finish="${2}/finish" scratch="${2}" sha file number ledger_earlier="" @@ -114,6 +116,7 @@ fct_test_finish() { git -C "${finish}" commit -qm 'readiness review' fct_finish 0 fct_finish_said 'finish: no line or finding is open' + fct_test_finish_matrix # A change after the review needs a new review. printf 'A change after the review.\n' >>"${finish}/README.md" diff --git a/sdk/extensionrepo/finish.go b/sdk/extensionrepo/finish.go index a3fda262..7d064427 100644 --- a/sdk/extensionrepo/finish.go +++ b/sdk/extensionrepo/finish.go @@ -16,6 +16,9 @@ import ( func finishRepository() int { problems, spec := formProblems() specs, limits := specProblems(spec) + if spec != "" { + specs = append(specs, matrixProblems(spec, limits)...) + } problems = append(append(problems, specs...), readinessProblems(spec, limits)...) for _, problem := range problems { fmt.Fprintln(os.Stderr, problem) diff --git a/sdk/extensionrepo/spec_matrix.go b/sdk/extensionrepo/spec_matrix.go new file mode 100644 index 00000000..489907ec --- /dev/null +++ b/sdk/extensionrepo/spec_matrix.go @@ -0,0 +1,86 @@ +package extensionrepo + +import ( + "fmt" + "maps" + "regexp" + "slices" + "strings" +) + +// matrixColumns are the columns of the table under ## Case matrix of spec.md. +// Each row is one family of cases. Accepted names the spec IDs of the inputs +// that pass, and Refused those of the inputs that the command refuses, or +// None (#423). +var matrixColumns = []string{"Family", "Source", "Accepted", "Refused"} + +var matrixID = regexp.MustCompile("`([a-z][a-z0-9]*(?:-[a-z0-9]+)*)`") + +// matrixProblems reports a family of ## Case matrix without a test, and a +// line of Failures or Acceptance in no family. Each spec ID of a family must +// be a line of Failures or Acceptance, or a limit that the user accepted. The +// test kit requires a test that claims each such line. Each family names a +// spec ID that passes, so each refusal has its pair. +func matrixProblems(spec string, limits []limit) []string { + const fix = "Fix: give spec.md a table under ## Case matrix with the columns Family, Source, Accepted, and Refused, one row for each family of cases" + proven := map[string]int{} + for _, l := range SpecLines(spec) { + if l.ID != "" && (l.Part == "Failures" || l.Part == "Acceptance") { + proven[l.ID] = l.Number + } + } + var problems []string + named, header := map[string]bool{}, false + for _, l := range SpecLines(spec) { + if l.Part != "Case matrix" { + continue + } + switch { + case l.Kind == SpecHeader: + header = slices.Equal(l.Cells, matrixColumns) + if !header { + problems = append(problems, fmt.Sprintf("spec.md:%d: the table of ## Case matrix has other columns. %s", l.Number, fix)) + } + case l.Kind == SpecRow && header && len(l.Cells) == len(matrixColumns): + accepted, refused := matrixIDs(l.Cells[2]), matrixIDs(l.Cells[3]) + at := fmt.Sprintf("spec.md:%d: the family %q ", l.Number, l.Cells[0]) + if len(accepted) == 0 { + problems = append(problems, at+"names no spec ID of an input that passes. "+ + "Fix: name in Accepted the line whose test gives an input that passes, so each refusal has its pair") + } + if len(refused) == 0 && strings.TrimSuffix(l.Cells[3], ".") != "None" { + problems = append(problems, at+"names no spec ID of a refused input, and not None. Fix: name in Refused the line whose test refuses an input, or write None") + } + for _, id := range append(accepted, refused...) { + named[id] = true + _, found := proven[id] + if !found && !slices.ContainsFunc(limits, func(l limit) bool { return l.id == id && l.accepted }) { + problems = append(problems, fmt.Sprintf("%snames `%s`, which is no line of Failures or Acceptance and no accepted limit, so no test proves it. "+ + "Fix: prove it with a test in a round, or ask the user to accept it as a limit", at, id)) + } + } + case l.Kind == SpecRow: + problems = append(problems, fmt.Sprintf("spec.md:%d: this row of ## Case matrix has no cell for each column. %s", l.Number, fix)) + } + } + if !header && len(problems) == 0 { + return []string{"spec.md: it holds no table under ## Case matrix, so no list of the families of cases shows that each has a test. " + fix} + } + ids := slices.SortedFunc(maps.Keys(proven), func(a, b string) int { return proven[a] - proven[b] }) + for _, id := range ids { + if !named[id] { + problems = append(problems, fmt.Sprintf("spec.md:%d: the line `%s` is in no family of ## Case matrix. "+ + "Fix: name it in the row of its family, beside a spec ID of the other outcome", proven[id], id)) + } + } + return problems +} + +// matrixIDs returns the spec IDs in backticks of a cell. +func matrixIDs(cell string) []string { + var ids []string + for _, m := range matrixID.FindAllStringSubmatch(cell, -1) { + ids = append(ids, m[1]) + } + return ids +} diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index 61c8f2aa..46e73cab 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -64,7 +64,7 @@ Each round ends with the independent review, the readiness review, and `just fin Do these steps in this order. Commit each of the first four steps on its own: 1. Record the intake facts under `## Intake` in `vision.md`, as "Evidence of the intake" in `intake.md` says. The facts include the effects, the exit codes, the resource use, and the known failures. A fact that no source gives stays an open line of `spec.md`. -2. Write every line of the command in `spec.md`, from the generic cases of the contract and the intake. Each line that has no test stays `open`, under `## Open lines`. +2. Write every line of the command in `spec.md`, from the generic cases of the contract and the intake. Each line that has no test stays `open`, under `## Open lines`. Write the table under `## Case matrix` too, as "Case matrix" says. 3. Write the first scenario at the public command boundary, with its claim `# spec: `. Commit it while it fails. Run it at that commit with `just scenario `. Keep that command, its exit status, and the line of the scenario that failed. A compile error is no such failure. `just signoff` runs it again. When the scenario cannot run without the command, commit a handler that does no work in this step. Then the run reaches the assertion. 4. Change the handler until that scenario passes, with "Change a command" in `AGENTS.md`. 5. When no line stays open, write `None.` under `## Open lines`, with a sentence that cites the generic cases. @@ -76,11 +76,22 @@ Give the commits of steps 1 to 4 to the reviewer, who records them under `## Ord Give the reviewer the record of the run of step 3 too. The reviewer checks it, and records it under `## First failing run`. The first round is incomplete by design. The open lines start the next round. +### Case matrix + +Before the review of round 1, `## Case matrix` of `spec.md` lists each family of cases of each command (#423): + +- One row for each generic case that applies, and for each family that the intake found +- A family groups the inputs that one rule reads, such as each way that the input declares how to read it +- Each refusal and each branch that classifies an input has a pair of tests: one input passes, and one input is refused +- A repair of a finding adds a pair of tests for its whole family, not only a test of the reported example + +`just finish` fails when a family names no test, or when a line of Failures or Acceptance is in no family. + ### Each next round 1. Select open lines, and the findings that the ledger records as `open`. 2. Move each line from `## Open lines` into Failures or Acceptance. -3. For each line, write a scenario with the claim `# spec: `. It fails first, for the missing behavior. +3. For each line, write a scenario with the claim `# spec: `. It fails first, for the missing behavior. For a finding, test its whole family with a pair, as "Case matrix" says. 4. Change the handler until the scenario passes. The line is then `proven`. 5. End the round. @@ -99,6 +110,7 @@ The first round is incomplete by design. The open lines start the next round. ### Reach "Done when" in 5 rounds - Round 1 includes a test file for each generic case that applies +- Ask the first reviewer to check that `## Case matrix` is complete and to run it, before it reviews any repair - Ask the first reviewer to test families of cases, such as all the declarations that change the meaning, not one counterexample - Fix a root cause, such as a parser or a conversion rule, before the next review - Keep the last rounds for verification, not for one patch per warning diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 8274eef0..249aac13 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -83,7 +83,14 @@ A loss that you report but that the user did not accept is a `warn` finding, not ## Round 1 -In round 1, review the order of the work, not only the code: +In round 1, review the order of the work, not only the code. +First, before you review any repair, check `## Case matrix` of `spec.md` (#423): + +- Each generic case that applies, and each family that the intake found, has its row +- Each refusal and each branch that classifies an input has a pair of tests: one input passes, and one input is refused +- Run the tests of the matrix. A family that the matrix lacks is a finding + +Then review the rest of round 1: - The intake commit comes before the spec commit. The intake gives, for each fact, its source, its version, and the part that the author read, and the result of each experiment apart. A link alone is not evidence of a read - The record of the first failing run gives the command, its exit status, and the assertion that failed at the scenario commit. The command is `just scenario `. Run it again at that commit when in doubt, and `just signoff` runs it again too. Refuse a failure that is only a compile error, because it does not prove the missing behavior diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index 7ea5c186..3dc465b6 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -30,6 +30,10 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | Each case of the Output group that applies has a test file that the tests make, checked by an independent reader | Prose only | | The tests make their files; never search the home folder or copy a file of the user | Prose only | | Round 1 tests each generic case that applies, and the first review tests families of cases | Prose only | +| Before the review of round 1, `## Case matrix` gives each family its tests, and each line of Failures and Acceptance a family | [Finish matrix][finish-matrix] | +| Each refusal and each branch that classifies an input has a pair of tests | [Finish matrix][finish-matrix] needs a passing ID in each family; the reviewer checks the rest | +| The first reviewer checks that the matrix is complete and runs it, before any repair | Prose only | +| A repair adds a pair of tests for its whole family | Prose only | | A root cause is fixed before the next review, and the last rounds verify | Prose only | | Build each command with the `nicely-extension` skill, in rounds from its first pull request | Prose only | | The first round writes every line of `spec.md`; a line without a test stays `open` | Prose only | @@ -68,6 +72,7 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". [ledger-run]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger_run.go [ledger-findings]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger_findings.go [ledger-history]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/ledger_history.go +[finish-matrix]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/spec_matrix.go [finish-verdict]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/readiness_verdict.go [readiness]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/readiness.go [signoff]: https://github.com/pascalandy/nicely/blob/main/sdk/extensionrepo/signoff.go diff --git a/template/spec.md.jinja b/template/spec.md.jinja index 2a2bd569..6d7238f6 100644 --- a/template/spec.md.jinja +++ b/template/spec.md.jinja @@ -55,6 +55,24 @@ Fill: the real input that proves the command, such as a fixture of the registry Fill: each input that the wrapped program accepts but that the domain refuses, with its code. A scenario and `acceptance.sh` refuse each one. +## Case matrix + +Before the review of round 1, this table lists each family of cases of each command, with the spec IDs of its tests (#423). +A family comes from a generic case that applies, or from a family that the intake found. +An example of such a family is each way that the input declares how to read it. +Accepted names the lines whose tests give an input that passes. Refused names the lines whose tests refuse an input, or `None`. +Each refusal and each branch that classifies an input has a pair: a test that passes, and a test that refuses. +Each line of Failures and of Acceptance is in a family. `just finish` fails when a family has no test. + +| Family | Source | Accepted | Refused | +|---|---|---|---| +| The input is a regular file that exists | Input: Unreadable | `size`, `lines` | `bad-input` | +| The input holds text | Input: Of the wrong type | `lines` | `not-text` | +| `python3` runs the worker | Other: Missing program, and the intake | `size` | `no-python`, `worker-fails` | +| A signal stops the run | Interruption | `size` | `interrupted`, `terminated` | + +Fill: one row for each family of cases of your commands, in place of the rows of `inspect`. A repair of a finding adds a pair of tests for its whole family, not only a test of the reported example. + ## Open lines A round of the skill in `.agents/skills/nicely-extension/` moves an open line into Failures or Acceptance, with the scenario that claims it. From b987e58b400c6797161949c5430f8d19ff054bb4 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:28:41 -0400 Subject: [PATCH 48/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20template?= =?UTF-8?q?:=20the=20error=20mapping=20of=20the=20skill=20moves=20to=20its?= =?UTF-8?q?=20own=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The section "Error mapping" of the nicely-extension skill moves to error-mapping.md beside it, and the skill links it. Nothing changes in the rules, so the next rules of the rounds fit in SKILL.md. Co-Authored-By: Claude Opus 5.5 --- template/.agents/skills/nicely-extension/SKILL.md | 8 +------- .../.agents/skills/nicely-extension/error-mapping.md | 11 +++++++++++ 2 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 template/.agents/skills/nicely-extension/error-mapping.md diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index 46e73cab..900a7a7d 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -28,13 +28,7 @@ Read [intake.md](intake.md) before you write code. It gives the facts to get fro ## Error mapping -`sdk/wrap` answers each failure that "Wrapped programs" in the contract lists. The handler maps the rest: - -- Check each input before the program starts. Answer `USAGE_INVALID`, with a hint that shows a call that works -- Map each exit code that the spec names to its outcome. Return `Ended.Failed` for every other code, so no code passes through -- Decode the output into a typed value. Return `Ended.Malformed` when a field is missing or invalid, so a failure never becomes a success -- Never give exit 75 for the program's own 75. Only the handler can prove that a repeat is safe, as "Retry safety" in the contract says -- Put each message and hint in `locales/en.toml`, never in code +Read [error-mapping.md](error-mapping.md) before you map the failures of the wrapped program. It says which failures `sdk/wrap` answers, and how the handler maps the rest. ## Acceptance lines diff --git a/template/.agents/skills/nicely-extension/error-mapping.md b/template/.agents/skills/nicely-extension/error-mapping.md new file mode 100644 index 00000000..4dbe009b --- /dev/null +++ b/template/.agents/skills/nicely-extension/error-mapping.md @@ -0,0 +1,11 @@ +# Error mapping + +This file says how a handler maps the failures of the wrapped program. + +`sdk/wrap` answers each failure that "Wrapped programs" in the contract lists. The handler maps the rest: + +- Check each input before the program starts. Answer `USAGE_INVALID`, with a hint that shows a call that works +- Map each exit code that the spec names to its outcome. Return `Ended.Failed` for every other code, so no code passes through +- Decode the output into a typed value. Return `Ended.Malformed` when a field is missing or invalid, so a failure never becomes a success +- Never give exit 75 for the program's own 75. Only the handler can prove that a repeat is safe, as "Retry safety" in the contract says +- Put each message and hint in `locales/en.toml`, never in code From 7bf5dc11a55fd1e72f05826ccfffd30d22b82b9b Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:29:02 -0400 Subject: [PATCH 49/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20template:=20each?= =?UTF-8?q?=20fact=20of=20the=20intake=20links=20the=20part=20that=20the?= =?UTF-8?q?=20agent=20read?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An intake could name a part of a source with no read of it in the transcript. Each settled fact now links the part of the source that the agent read, at a fixed revision, and a fact without a read is marked inference. Each unknown fact is an open line with an ID before the spec. The reviewer of round 1 refuses a fact that only cites a source. Refs #423 Co-Authored-By: Claude Opus 5.5 --- template/.agents/skills/nicely-extension/intake.md | 7 ++++--- .../.agents/skills/nicely-extension/readiness-review.md | 2 ++ template/AGENTS/rule-index.md.jinja | 3 +++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/template/.agents/skills/nicely-extension/intake.md b/template/.agents/skills/nicely-extension/intake.md index 0b88bde5..c8dd4551 100644 --- a/template/.agents/skills/nicely-extension/intake.md +++ b/template/.agents/skills/nicely-extension/intake.md @@ -28,9 +28,10 @@ Write each fact that the documentation does not give as an open line of `spec.md The intake records what you read, not only where the facts are (#420): -- For each fact: the source, its version or revision, and the part that you read. Such a part is a function of the script or a section of the manual page +- For each settled fact: a link to the part of the source that you read, at a fixed revision. Such a part is a function of the script at a commit, or a section of the manual page of a version (#423) +- Each fact that you did not read in a source, marked `inference` - The result of each experiment, apart from the documented facts, with its command and what it gave - Each declaration of the input that changes how a reader shows the content, not only the file types. `generic-cases.md` in the contract says why -- Each fact that no source gives, as an open line of `spec.md` +- Each fact that no source gives, as an open line of `spec.md` with its ID, before the spec commit -A link alone is not evidence of a read. Read the source first. Then run experiments only to confirm what you read, with a command that neither writes nor bills. +A link alone is not evidence of a read. A reviewer of round 1 refuses a fact that only cites a source. Read the source first. Then run experiments only to confirm what you read, with a command that neither writes nor bills. diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 249aac13..36554293 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -93,6 +93,8 @@ First, before you review any repair, check `## Case matrix` of `spec.md` (#423): Then review the rest of round 1: - The intake commit comes before the spec commit. The intake gives, for each fact, its source, its version, and the part that the author read, and the result of each experiment apart. A link alone is not evidence of a read +- Refuse a fact that only cites a source, such as a link or a file name, with no part read. Such a fact is marked `inference`, or it is a finding +- Each unknown fact is an open line of `spec.md` with an ID, in the spec commit - The record of the first failing run gives the command, its exit status, and the assertion that failed at the scenario commit. The command is `just scenario `. Run it again at that commit when in doubt, and `just signoff` runs it again too. Refuse a failure that is only a compile error, because it does not prove the missing behavior - Each case of the Output group of `generic-cases.md` that applies has a test file that the tests make, and an independent check. A case name alone is not a test - Test families of cases, such as all the declarations that change the meaning, not one counterexample diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index 3dc465b6..7e68d86e 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -27,6 +27,9 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | `just signoff` runs the first scenario again at its commit, and never runs the text of the record | [Signoff replay][signoff-replay] | | A compile error is no failure of the first scenario; a scenario that cannot run without the command commits a handler that does no work | Prose only | | Read the source first, then run experiments only to confirm what you read | Prose only | +| Each settled fact of the intake links the part read, at a fixed revision; a fact without a read is marked `inference` | Prose only | +| Each unknown fact of the intake is an open line with an ID before the spec | Prose only | +| The reviewer of round 1 refuses a fact that only cites a source | Prose only | | Each case of the Output group that applies has a test file that the tests make, checked by an independent reader | Prose only | | The tests make their files; never search the home folder or copy a file of the user | Prose only | | Round 1 tests each generic case that applies, and the first review tests families of cases | Prose only | From c415bd80dc46af1ccbdd4544116f65b626e82c37 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:29:02 -0400 Subject: [PATCH 50/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20template:=20a=20te?= =?UTF-8?q?st=20file=20with=20private=20text=20in=20a=20block=20that=20a?= =?UTF-8?q?=20reader=20needs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No build had a test file for this case before its first review. Before the first review, acceptance.sh now makes a file whose block that a reader needs uses a value that is not the default and holds private text. An independent reader proves that the private text is gone and that the content still shows correctly, and a handler that keeps the raw block fails the check. Each experiment uses a new scratch folder, and no recursive delete runs inside sh -c or a loop, which a harness can stop for an approval that nobody gives. The tests and just acceptance need no root, no mount, and no system service. Refs #423 Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/generic-cases.md | 2 +- docs/north-star/contract/test-kit.md | 4 +++- .../.agents/skills/nicely-extension/SKILL.md | 10 ++++++++++ template/AGENTS/rule-index.md.jinja | 4 ++++ template/acceptance.sh.jinja | 17 +++++++++++------ 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/docs/north-star/contract/generic-cases.md b/docs/north-star/contract/generic-cases.md index 09b92bbd..a2e0ef36 100644 --- a/docs/north-star/contract/generic-cases.md +++ b/docs/north-star/contract/generic-cases.md @@ -29,7 +29,7 @@ An error that names the input gives its path or its item, so an agent knows whic | Output | Appears during the run, without `--force` | Another process writes `notes.txt` after the check and before the write | The command never replaces that file. It publishes with a create that fails on an existing file, such as a hard link. When the file system cannot create the output without a replace, the command fails. It never uses a check and then a rename that replaces | | Output | Temporary file | A link `notes.txt.tmp` to `notes.md`, made before the run | The command creates its temporary file with exclusive create, and with a name that no other process can predict. It never writes through a link at that path | | Output | Keeps private data of the input | `notes.md` starts with the line `author: Ana Lima`, and `notes.txt` keeps that name | The command removes the private data, or the user accepts a limit | -| Output | Keeps data that a reader needs | `notes.md` starts with a valid header that a reader needs, with the line `encoding: latin-1`, and that header also holds the line `author: Ana Lima` | The command keeps the data that the reader needs to show the content correctly, and removes the private text. The test proves both: `Ana Lima` is not in the output, and a reader still shows the content correctly. Otherwise, the user accepts a limit | +| Output | Keeps data that a reader needs | `notes.md` starts with a valid header that a reader needs, with the line `encoding: latin-1`, and that header also holds the line `author: Ana Lima` | The command keeps the data that the reader needs to show the content correctly, and removes the private text. The test proves both: `Ana Lima` is not in the output, and a reader still shows the content correctly. The header uses a value that is not the default, and the test file exists before the first review. Otherwise, the user accepts a limit | | Output | Loses data that changes the meaning | `notes.md` declares the encoding `latin-1`, and `notes.txt` keeps its bytes without that line | The output keeps the declaration, such as a text encoding or a unit, or the command applies it to the content. The intake lists each declaration that changes how a reader shows the content, not only the file types. The test file uses a value that is not the default, and the independent reference applies that declaration before it compares the content | | Interruption | SIGINT or SIGTERM during the work | Ctrl-C while `inspect` counts the lines of a large file | Exit 130 or 143, with an answer that names the input. No partial output remains, the old output stays, and no child process remains | | Interruption | Timeout | `inspect --timeout 1ms` on a large file | The timeout outcome, with an answer that names the input, even when the check of the program uses up the deadline. No partial output remains | diff --git a/docs/north-star/contract/test-kit.md b/docs/north-star/contract/test-kit.md index f427043e..e546d3f2 100644 --- a/docs/north-star/contract/test-kit.md +++ b/docs/north-star/contract/test-kit.md @@ -55,10 +55,12 @@ The check compares the content that this reader gives with an independent refere Each case of the Output group of [Generic cases](generic-cases.md) that applies to the command needs a test file (#420). `acceptance.sh` makes that file in the extension's own domain, and proves the safe outcome on it. Two cases need more: -- **Keeps data that a reader needs.** The test file holds a valid block that a reader needs, and that block holds private text. The check proves both results: the private text is not in the output, and the output still shows the content correctly +- **Keeps data that a reader needs.** The test file holds a valid block that a reader needs, and that block holds private text. The block uses a value that is not the default. The check proves both results with an independent reader: the private text is not in the output, and the output still shows the content correctly. A handler that keeps the raw block fails the check (#423) - **Loses data that changes the meaning.** Some declarations change how a reader shows the content. The test file gives each of them a value that is not the default. The independent reference applies that declaration before it compares the content, so a lost declaration fails the check The tests make each file that they use. They never copy a file of the user, and the author never searches the home folder of the user for sample files. +The tests make the file of each case above before the first review. +The tests and `just acceptance` need no root, no mount, and no system service. Other scenario runners use `NewCoverage`, `Coverage.Observe`, and `Coverage.Missing` for the same answer and coverage checks. `Observe` receives completed calls, including their exit code and both streams, without changing process execution or output. diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index 900a7a7d..a363f091 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -44,6 +44,16 @@ A case name is not a test. Each case of the Output group that applies gets a tes ## Test files The tests of the extension use files that the tests make, such as a text file that a scenario or `acceptance.sh` writes. + +Before the first review, `acceptance.sh` makes a test file with a block that a reader needs, for each command that writes an output (#423): + +- The block uses a value that is not the default, such as `encoding: latin-1`, and it holds private text, such as `author: Ana Lima` +- An independent reader proves both results: the private text is not in the output, and the output still shows the content correctly +- Change the handler to keep the raw block, and check that `just acceptance` fails. Then undo that change + +The tests and `just acceptance` need no root, no mount, and no system service. +Each experiment uses a new scratch folder, such as one that `mktemp -d` makes. +Never delete files recursively inside `sh -c` or in a loop. A coding harness can stop such a command and wait for an approval that nobody gives. Do not search the home folder of the user for sample files. Never copy a file of the user into a test. ## Rounds diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index 7e68d86e..b5f21d09 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -30,6 +30,10 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | Each settled fact of the intake links the part read, at a fixed revision; a fact without a read is marked `inference` | Prose only | | Each unknown fact of the intake is an open line with an ID before the spec | Prose only | | The reviewer of round 1 refuses a fact that only cites a source | Prose only | +| Before the first review, a test file holds a block that a reader needs, with a value that is not the default and private text | Prose only | +| An independent reader proves that the private text is gone and the content shows correctly; a handler that keeps the raw block fails | Prose only | +| Each experiment uses a new scratch folder; never delete files recursively inside `sh -c` or a loop | Prose only | +| The tests and `just acceptance` need no root, no mount, and no system service | Prose only | | Each case of the Output group that applies has a test file that the tests make, checked by an independent reader | Prose only | | The tests make their files; never search the home folder or copy a file of the user | Prose only | | Round 1 tests each generic case that applies, and the first review tests families of cases | Prose only | diff --git a/template/acceptance.sh.jinja b/template/acceptance.sh.jinja index 0bccd580..a05959e4 100644 --- a/template/acceptance.sh.jinja +++ b/template/acceptance.sh.jinja @@ -82,12 +82,17 @@ grep -qE "\"lines\":$(awk 'END { print NR }' "$input")[,}]" "$scratch/answer" || # Fill: for each case of the Output group of generic-cases.md in the contract # that applies to the command, make a test file of this domain here, and check # its safe outcome as "Acceptance test files" of test-kit.md says. This script -# makes each file. Never copy a file of the user. Such as: a header that a -# reader needs, with the lines encoding: latin-1 and author: Ana Lima. The -# output loses the name, and a reader still shows the same text. A declaration -# that changes the meaning gets a value that is not the default, and the -# reference applies it before the compare. inspect writes no output, so it has -# no such case. +# makes each file, before the first review. Never copy a file of the user. A +# declaration that changes the meaning gets a value that is not the default, +# and the reference applies it before the compare. inspect writes no output, +# so it has no such case. For a command that writes notes.txt from notes.md, +# the block that a reader needs can look like this: +# printf 'encoding: latin-1\nauthor: Ana Lima\n\ncaf\351\n' >"$work/header.md" +# (cd "$work" && ncly "${command[@]}" header.md --json --no-input) >/dev/null || die 'the header failed' +# ! grep -qF 'Ana Lima' "$work/header.txt" || die 'the output keeps the private text' +# printf 'caf\303\251\n' | cmp -s - <(reader "$work/header.txt") || die 'a reader shows other text' +# Here reader is an independent reader of the output. Change the handler to +# keep the raw header, and check that this part fails (#423). # A second run checks the Determinism criterion of qa.md. (cd "$work" && ncly "${command[@]}" "$input" --json --no-input) >"$scratch/second" || die "the second run failed" From 5ecbb96b4c8fbca529205f359564752209f4114b Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:44:58 -0400 Subject: [PATCH 51/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20an=20or?= =?UTF-8?q?dinary=20table=20cell=20that=20ends=20with=20block=20passes=20f?= =?UTF-8?q?inish?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finish failed on each table cell whose last word was a severity, so a cell such as "the header block" failed. A cell now grades in that form only in a row that names a finding ID, such as | R1-F1 | really a warn |. Elsewhere a cell grades only when it holds the severity alone. The level form needs a hyphen, such as warn-level, so "block level" in prose passes. Refs #422, #423 Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 5 +++-- scripts/template-finish-test.sh | 2 +- sdk/extensionrepo/readiness_severity.go | 20 ++++++++++--------- .../nicely-extension/readiness-review.md | 5 +++-- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index d27d0763..963a2315 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -29,9 +29,10 @@ The severities are the words block, warn, and advise, in any case. After the first three lines, outside the line `- **Severity**:` of a finding, finish fails on these forms, and only on these (#413, #416, #420, #422): - A severity in code, such as `` `warn` ``, also in fenced code -- A cell of a table whose last word is a severity, with only marks after it. Examples are `` `warn` ``, `**block**`, `(advise)`, and `really a warn` +- A cell of a table that holds only a severity, with only marks around it, such as `` `warn` ``, `**block**`, or `(advise)` +- In a row that names a finding ID, a cell whose last word is a severity, such as `| R1-F1 | really a warn |` - The word severity, grade, grades, or graded, then only marks or spaces, then a severity, such as "Severity: block" or "Grade: warn" -- A severity, then a hyphen or a space, then the word level, such as "warn-level" +- A severity, then a hyphen and the word level, such as "warn-level". The text "block-level" fails too, so rephrase it - Under `## Findings`, the word severity alone Finish reads each other line as prose, and it passes. This is also true when the line grades in other words, such as "the grade is warn" or "a block for me". diff --git a/scripts/template-finish-test.sh b/scripts/template-finish-test.sh index 51b4fa5f..aaffbf52 100644 --- a/scripts/template-finish-test.sh +++ b/scripts/template-finish-test.sh @@ -111,7 +111,7 @@ fct_test_finish() { fct_finish 1 fct_finish_said 'readiness.md: its first three lines hold no line Commit: or Verdict: ' - fct_readiness "${sha}" 'Ready with stated limits' "${warn}"$'\n- **State**: `limit`\n- **Spec**: `large-file`\n\n'"${advise}"$'\n- **State**: `declined`\n- **Reason**: the answer stays small\n\n## Severity assessment\n\nEach finding names its severity.\n\n## Notes\n\n| R1-F2 | the answer stays small |\n\nLarge files block the terminal, and inspect may warn.' + fct_readiness "${sha}" 'Ready with stated limits' "${warn}"$'\n- **State**: `limit`\n- **Spec**: `large-file`\n\n'"${advise}"$'\n- **State**: `declined`\n- **Reason**: the answer stays small\n\n## Severity assessment\n\nEach finding names its severity.\n\n## Notes\n\n| R1-F2 | the answer stays small |\n| Input | the header block |\n\nLarge files block the terminal, and inspect may warn.' git -C "${finish}" add readiness.md git -C "${finish}" commit -qm 'readiness review' fct_finish 0 diff --git a/sdk/extensionrepo/readiness_severity.go b/sdk/extensionrepo/readiness_severity.go index 14d2aa27..abf7f15e 100644 --- a/sdk/extensionrepo/readiness_severity.go +++ b/sdk/extensionrepo/readiness_severity.go @@ -17,9 +17,9 @@ var ( // any part. Only marks can stand between them, such as in "Severity: // block" or "Grade: warn" (#420, #422). readinessGrade = regexp.MustCompile(`(?i)\b(severity|grade[sd]?)\b[^a-z0-9]*\b(block|warn|advise)\b`) - // readinessLevel finds a severity before the word level, such as - // "warn-level" or "block level", in any part (#422). - readinessLevel = regexp.MustCompile(`(?i)\b(block|warn|advise)[- ]level\b`) + // readinessLevel finds a severity, a hyphen, and the word level, such as + // "warn-level", in any part (#422). + readinessLevel = regexp.MustCompile(`(?i)\b(block|warn|advise)-level\b`) // readinessWord finds the word severity under ## Findings (#413). readinessWord = regexp.MustCompile(`(?i)\bseverity\b`) ) @@ -45,18 +45,20 @@ func severityCause(text, fence string) string { return "this line names the word severity outside the line - **Severity**: of a finding. Fix: grade each finding only in its line - **Severity**:, and rephrase this line without the word severity" } -// severityCell returns the severity that ends a cell of a table row, with -// only marks after it, such as `warn`, **block**, or really a warn. It -// returns "" for any other line. So the cell "inputs block the terminal" -// passes (#416, #420, #422). +// severityCell returns the severity of a cell of a table row. Any row can +// hold it alone, with only marks around it, such as `warn` or **block**. A +// row that names a finding ID can also end a cell with it, such as +// | R1-F1 | really a warn |. It returns "" for any other line. So the cell +// "the header block" passes (#416, #420, #422). func severityCell(text string) string { trimmed := strings.TrimSpace(text) if !strings.HasPrefix(trimmed, "|") { return "" } + finding := readinessMention.MatchString(trimmed) for _, cell := range strings.Split(trimmed, "|") { - words := strings.Fields(strings.ToLower(cell)) - if len(words) == 0 { + words := strings.Fields(strings.ToLower(strings.Trim(cell, " \t`*_().:!"))) + if len(words) == 0 || len(words) > 1 && !finding { continue } if word := strings.Trim(words[len(words)-1], "`*_().:!"); severityRank[word] > 0 { diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 36554293..5e008b13 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -40,9 +40,10 @@ Name a severity only in the bullet **Severity** of its finding. `just finish` re After the first three lines, it fails on each form that grades in another place: - A severity in code, such as `` `warn` `` -- A cell of a table whose last word is a severity, such as `really a warn` +- A cell of a table that holds only a severity +- In a row with a finding ID, a cell that ends with a severity, such as `| R1-F1 | really a warn |` - The word severity or grade before a severity, such as "Severity: block" or "Grade: warn" -- A severity before the word level, such as "warn-level" +- A severity before `-level`, such as "warn-level" - A finding heading, such as `### R1-F2`, outside `## Findings`, or an ID outside it of no finding under it - The word severity under `## Findings` From 84117d6515728028628e49b3a8d72e8fe7f539b3 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:45:00 -0400 Subject: [PATCH 52/61] =?UTF-8?q?=E2=9C=85=20test:=20template:=20the=20rep?= =?UTF-8?q?lay=20refuses=20another=20failing=20line,=20and=20just=20scenar?= =?UTF-8?q?io=20runs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A test now records one line of the first scenario while the replay fails on another, and signoff refuses it. The template test runs just scenario on a scenario that passes and on a name that is a path. The test kit skips coverage only when -run selects the scenario that NCLY_TEST_SCENARIO names, and test-kit.md documents that variable. Refs #422 Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/test-kit.md | 1 + scripts/template-signoff-test.sh | 27 +++++++++++++++++++++------ sdk/extensionrepo/scenario.go | 12 +++++++++--- sdk/extensiontest/coverage_report.go | 7 +++++-- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/docs/north-star/contract/test-kit.md b/docs/north-star/contract/test-kit.md index e546d3f2..502e04cc 100644 --- a/docs/north-star/contract/test-kit.md +++ b/docs/north-star/contract/test-kit.md @@ -32,6 +32,7 @@ A scratch `HOME` does not hide the OS keychain. `ncly` reads it only when the va - Each runnable command has a recorded call through `ncly`, `ncly-`, `exits`, `signal`, `appears`, or `refuses` after its answer passes validation - A command whose declared result kind is `text` counts a completed success, since that kind has no JSON answer - Coverage runs after all scenarios finish. Comments, assertions, skipped conditions, help, and shell or background `exec` calls earn no coverage +- `just scenario ` runs one scenario. It sets `NCLY_TEST_SCENARIO` to that name, and selects only that scenario with `-run`. The kit then skips coverage, which needs every scenario (#422) - Human text earns no coverage for a command with a structured result - A path-qualified Nicely call in `exits` is refused with the registered command spelling - A failed report on stdout is checked as one complete object, including its envelope and declared result fields diff --git a/scripts/template-signoff-test.sh b/scripts/template-signoff-test.sh index e33959b6..616ce852 100644 --- a/scripts/template-signoff-test.sh +++ b/scripts/template-signoff-test.sh @@ -34,6 +34,10 @@ SCRIPT fct_signoff_refuses "${scratch}" "${scratch}/signoff-compile" 'just scenario first' 'before any line of the scenario, such as on a compile error' fct_signoff_ready "${repo}" "${scratch}" "${scratch}/signoff-passes" inspect fct_signoff_refuses "${scratch}" "${scratch}/signoff-passes" 'just scenario first' 'passes at its Scenario commit' + # The run must fail on the line that the record names, not on another line of the scenario. + fct_signoff_ready "${repo}" "${scratch}" "${scratch}/signoff-line" read + fct_signoff_refuses "${scratch}" "${scratch}/signoff-line" 'just scenario first' 'and ## First failing run names the assertion' \ + "$(grep -m 1 '^refuses ' "${scratch}/signoff-line/testdata/script/first.txtar")" fct_signoff_ready "${repo}" "${scratch}" "${ready}" read fct_signoff_review "${ready}" 'just scenario first' @@ -45,6 +49,16 @@ SCRIPT fct_has "${scratch}/signoff.out" "Signed off on ${sha}" grep -qF 'signoff: the first scenario fails again at' "${scratch}/signoff.out" || fct_fail 'just signoff did not run the first failing run again' fct_has "${scratch}/signed-sha" "${sha}" + + # just scenario runs one scenario, without the coverage of the others, and refuses a name of no scenario. + (cd "${ready}" && just scenario first) >"${scratch}/scenario.out" 2>&1 || { + cat "${scratch}/scenario.out" >&2 + fct_fail 'just scenario first fails on a scenario that passes' + } + if (cd "${ready}" && just scenario ../first) >"${scratch}/scenario.out" 2>&1 || ! grep -qF '"../first" is no name of a scenario' "${scratch}/scenario.out"; then + cat "${scratch}/scenario.out" >&2 + fct_fail 'just scenario accepts a path as the name of a scenario' + fi } # fct_signoff_ready copies the rendered repository $1 to $3, with the steps of @@ -79,10 +93,11 @@ fct_signoff_ready() { } # fct_signoff_review commits the readiness review of the head of $1, whose -# first failing run names the command $2. +# first failing run names the command $2, and the assertion $3 or the line +# stderr of the first scenario. fct_signoff_review() { - local expected - expected="$(grep -m 1 '^stderr ' "${1}/testdata/script/first.txtar")" + local expected="${3:-}" + [[ -n "${expected}" ]] || expected="$(grep -m 1 '^stderr ' "${1}/testdata/script/first.txtar")" finish="${1}" ledger_run=$'- **Commit**: '"$(fct_step "${1}" scenario)"$'\n- **Command**: `'"${2}"$'`\n- **Exit**: 1\n- **Assertion**: `'"${expected}"'` found no match' \ fct_readiness "$(git -C "${1}" rev-parse HEAD)" Ready None. git -C "${1}" add readiness.md @@ -90,11 +105,11 @@ fct_signoff_review() { } # fct_signoff_refuses requires that just signoff of the repository $2, whose -# first failing run names the command $3, fails with the text $4 and signs -# nothing. +# first failing run names the command $3 and the assertion $5, fails with the +# text $4 and signs nothing. fct_signoff_refuses() { local status=0 - fct_signoff_review "${2}" "${3}" + fct_signoff_review "${2}" "${3}" "${5:-}" fct_signoff "${2}" "${1}" || status=$? if [[ "${status}" == 0 ]] || ! grep -qF -- "${4}" "${1}/signoff.out"; then cat "${1}/signoff.out" >&2 diff --git a/sdk/extensionrepo/scenario.go b/sdk/extensionrepo/scenario.go index 8d4854d5..1711caa7 100644 --- a/sdk/extensionrepo/scenario.go +++ b/sdk/extensionrepo/scenario.go @@ -8,10 +8,16 @@ import ( ) // ScenarioEnv names the scenario that a run of go test runs alone, for just -// scenario and for the repeat of signoff. The test kit then skips its -// coverage checks, which need every scenario. +// scenario and for the repeat of signoff. When -run is ScenarioRun of that +// name, the test kit skips its coverage checks, which need every scenario. const ScenarioEnv = "NCLY_TEST_SCENARIO" +// ScenarioRun is the -run pattern of go test that selects only the scenario +// testdata/script/.txtar. +func ScenarioRun(name string) string { + return "^TestExtension$/^" + name + "$" +} + // scenarioRecipe is the one command that ## First failing run can record: // just scenario . Finish and signoff read the name of the scenario from // it, and never run the text of the record (#422). @@ -21,7 +27,7 @@ var scenarioRecipe = regexp.MustCompile(`^just scenario ([A-Za-z0-9][A-Za-z0-9_- // testdata/script/.txtar. The test kit runs each scenario as a subtest // of TestExtension, as test-kit.md in the contract says. func scenarioArgs(name string) []string { - return []string{"-run", "^TestExtension$/^" + name + "$", "./..."} + return []string{"-run", ScenarioRun(name), "./..."} } // runScenario runs just scenario , through Test. diff --git a/sdk/extensiontest/coverage_report.go b/sdk/extensiontest/coverage_report.go index e9a6ce74..49eddc96 100644 --- a/sdk/extensiontest/coverage_report.go +++ b/sdk/extensiontest/coverage_report.go @@ -1,6 +1,7 @@ package extensiontest import ( + "flag" "os" "testing" @@ -12,8 +13,10 @@ import ( // scenarios finish. A run of one scenario, such as just scenario, skips it, // because coverage needs every scenario (#422). func (k *kit) reportCoverage(t *testing.T) { - if os.Getenv(extensionrepo.ScenarioEnv) != "" { - return + if name := os.Getenv(extensionrepo.ScenarioEnv); name != "" { + if run := flag.Lookup("test.run"); run != nil && run.Value.String() == extensionrepo.ScenarioRun(name) { + return + } } for _, reservation := range k.Reservations() { if err := extensionrepo.Notice(reservation); err != nil { From 171c79c7abf7048e00eb14dedb47ee54584746d8 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:45:02 -0400 Subject: [PATCH 53/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20a=20sta?= =?UTF-8?q?teless=20advise=20can=20be=20declined,=20and=20other=20review?= =?UTF-8?q?=20notes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The message of a finding of this round with no state now says that the author can decline an advise, and so does the skill. The rule index says that the matrix check needs a spec ID in Accepted, and the reviewer checks that it passes. A test helper of ncly gives its schema a URN too, so the tests of Nicely pass in a folder whose path holds %. A comment of ledger_run.go splits a sentence of 33 words. Refs #423 Co-Authored-By: Claude Opus 5.5 --- cmd/ncly/scriptcmds_test.go | 4 ++-- sdk/extensionrepo/ledger_findings.go | 6 +++++- sdk/extensionrepo/ledger_run.go | 7 ++++--- template/.agents/skills/nicely-extension/SKILL.md | 2 +- template/AGENTS/rule-index.md.jinja | 4 ++-- 5 files changed, 14 insertions(+), 9 deletions(-) diff --git a/cmd/ncly/scriptcmds_test.go b/cmd/ncly/scriptcmds_test.go index 45a78099..1de08d4a 100644 --- a/cmd/ncly/scriptcmds_test.go +++ b/cmd/ncly/scriptcmds_test.go @@ -558,8 +558,8 @@ func cmdSchema(ts *testscript.TestScript, neg bool, args []string) { compiler := jsonschema.NewCompiler() compiler.DefaultDraft(jsonschema.Draft2020) compiler.UseLoader(noRemote{}) - ts.Check(compiler.AddResource("describe.json", doc)) - schema, err := compiler.Compile("describe.json") + ts.Check(compiler.AddResource("urn:describe", doc)) + schema, err := compiler.Compile("urn:describe") ts.Check(err) answer, err := jsonschema.UnmarshalJSON(strings.NewReader(ts.ReadFile(args[0]))) ts.Check(err) diff --git a/sdk/extensionrepo/ledger_findings.go b/sdk/extensionrepo/ledger_findings.go index 4be735ee..de6fec2f 100644 --- a/sdk/extensionrepo/ledger_findings.go +++ b/sdk/extensionrepo/ledger_findings.go @@ -47,7 +47,11 @@ func stateProblems(findings []finding, n int, spec string, limits []limit) []str problems = append(problems, fmt.Sprintf("%sthe finding %s of round %d has no state. Fix: the reviewer of this round gives it the state open, fixed, limit, or declined", at, f.id, first)) continue } - problems = append(problems, fmt.Sprintf("%sthe finding %s of this round has no state, so it is open, even if ## Open lines holds None. Fix: prove its spec ID in a round, or ask the user to accept it as a limit", at, f.id)) + fix := "Fix: prove its spec ID in a round, or ask the user to accept it as a limit" + if f.severity == "advise" { + fix += ". The author can also decline an advise, with - **State**: `declined` and - **Reason**:" + } + problems = append(problems, fmt.Sprintf("%sthe finding %s of this round has no state, so it is open, even if ## Open lines holds None. %s", at, f.id, fix)) problems = append(problems, openLineProblems(f.id, open)...) case "open": problems = append(problems, fmt.Sprintf("%sthe finding %s is open, even if ## Open lines holds None. Fix: prove its spec ID in a round, or ask the user to accept it as a limit", at, f.id)) diff --git a/sdk/extensionrepo/ledger_run.go b/sdk/extensionrepo/ledger_run.go index 2f9d07c6..d55de53a 100644 --- a/sdk/extensionrepo/ledger_run.go +++ b/sdk/extensionrepo/ledger_run.go @@ -15,9 +15,10 @@ var firstRun = regexp.MustCompile(`^\s*[-*+]\s+\*\*(Commit|Command|Exit|Assertio // firstRunProblems checks the record of the first failing run of the current // build. It needs the Scenario commit of ## Order of round 1, so a record of -// an earlier build never stands in. It also needs the command just scenario -// of a scenario of that commit, an exit status that is not 0, and the -// assertion that failed, which holds a line of that scenario. just signoff +// an earlier build never stands in. It also needs three more lines. The +// command is just scenario , of a scenario of that commit. The exit +// status is not 0. The assertion that failed holds a line of that scenario. +// just signoff // runs the scenario again, and the reviewer judges whether the assertion // proves the missing behavior (#422). func firstRunProblems(readiness string, order map[string]round) []string { diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index a363f091..61121a7d 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -106,7 +106,7 @@ Before the review of round 1, `## Case matrix` of `spec.md` lists each family of 3. Run `just acceptance` on the real input until its independent checks pass. 4. Ask an agent that did not write the round to review the changes of the round. Give it the commits of the round. 5. Ask an agent that did not write the code to run the readiness review. It records the review and the ledger in `readiness.md`. One independent pass can do both reviews. -6. Write each finding that a test can prove as an open line, which names the finding by its ID. Each finding that the ledger records as `open` keeps its line under `## Open lines`, so `spec.md` and the ledger agree. A finding of this round with no state is `open` too. +6. Write each finding that a test can prove as an open line, which names the finding by its ID. Each finding that the ledger records as `open` keeps its line under `## Open lines`, so `spec.md` and the ledger agree. A finding of this round with no state is `open` too. The author can decline an `advise` instead, with a reason. 7. Run `just finish`. Stop as "Done when" in `AGENTS.md` says. `just finish` fails until the loop is done. 8. When `just finish` fails after round 5, stop, as "Round limit" says. 9. Otherwise, start the next round from the open lines and the open findings. Add its commits to the open pull request, or open a new one. Base a new pull request on the branch of the previous one when that one is not merged. diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index b5f21d09..02191bea 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -38,7 +38,7 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | The tests make their files; never search the home folder or copy a file of the user | Prose only | | Round 1 tests each generic case that applies, and the first review tests families of cases | Prose only | | Before the review of round 1, `## Case matrix` gives each family its tests, and each line of Failures and Acceptance a family | [Finish matrix][finish-matrix] | -| Each refusal and each branch that classifies an input has a pair of tests | [Finish matrix][finish-matrix] needs a passing ID in each family; the reviewer checks the rest | +| Each refusal and each branch that classifies an input has a pair of tests | [Finish matrix][finish-matrix] needs a spec ID in Accepted of each family; the reviewer checks that it passes | | The first reviewer checks that the matrix is complete and runs it, before any repair | Prose only | | A repair adds a pair of tests for its whole family | Prose only | | A root cause is fixed before the next review, and the last rounds verify | Prose only | @@ -56,7 +56,7 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | A finding of the last round with no state is `open` | [Ledger findings][ledger-findings] | | A severity is read only in the line **Severity** of a finding; the plain words block, warn, and advise pass elsewhere | [Readiness findings][readiness] | | A finding heading stays under `## Findings`, and no other form grades | [Readiness findings][readiness] | -| Finish finds a grade in code, at the end of a cell, after severity or grade, or before level | [Readiness findings][readiness] | +| Finish finds a grade in code, in a cell, after severity or grade, or before `-level` | [Readiness findings][readiness] | | The reviewer moves each grade in another form, which the parser does not find, to the bullet **Severity** | Prose only | | A broken promise of `spec.md` or of the help text is at least `warn` at its first review | Prose only | | Each round keeps its row and has a committed review; a build ends only at a review that passes | [Ledger history][ledger-history] | From 9c94f6bb168395ffecb5fe0c796871d873957a7c Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:56:20 -0400 Subject: [PATCH 54/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20template:=20the=20?= =?UTF-8?q?reviewer=20marks=20a=20repair=20fixed=20only=20with=20a=20pair?= =?UTF-8?q?=20for=20its=20family?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit readiness-review.md now says that a finding is fixed only when its repair adds a pair of tests for its whole family, as ## Case matrix lists it. spec.md states that rule in its prose, not only on a Fill: line that the author deletes. Refs #423 Co-Authored-By: Claude Opus 5.5 --- template/.agents/skills/nicely-extension/readiness-review.md | 1 + template/AGENTS/rule-index.md.jinja | 2 +- template/spec.md.jinja | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 5e008b13..b81e0718 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -62,6 +62,7 @@ Grade it so in the round that first reports it. These examples set the grade: - **An output whose meaning changes.** `notes.txt` drops the line `encoding: latin-1` and keeps the bytes, so a reader shows other letters. At least `warn`. `block` when `spec.md` promises the same text Never lower the severity of a finding in a later round. +Give a finding the state `fixed` only when its repair adds a pair of tests for its whole family, as `## Case matrix` lists it (#423). A `warn` or `block` finding closes only as `fixed` or `limit`. Only an `advise` can be `declined`. A change of `spec.md` that narrows a promise to close a `warn` or `block` finding is a `limit`. Only the user can accept it. diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index 02191bea..d92ee1aa 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -40,7 +40,7 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | Before the review of round 1, `## Case matrix` gives each family its tests, and each line of Failures and Acceptance a family | [Finish matrix][finish-matrix] | | Each refusal and each branch that classifies an input has a pair of tests | [Finish matrix][finish-matrix] needs a spec ID in Accepted of each family; the reviewer checks that it passes | | The first reviewer checks that the matrix is complete and runs it, before any repair | Prose only | -| A repair adds a pair of tests for its whole family | Prose only | +| A repair adds a pair of tests for its whole family, and only then is its finding `fixed` | Prose only | | A root cause is fixed before the next review, and the last rounds verify | Prose only | | Build each command with the `nicely-extension` skill, in rounds from its first pull request | Prose only | | The first round writes every line of `spec.md`; a line without a test stays `open` | Prose only | diff --git a/template/spec.md.jinja b/template/spec.md.jinja index 6d7238f6..9e4d45c4 100644 --- a/template/spec.md.jinja +++ b/template/spec.md.jinja @@ -63,6 +63,7 @@ An example of such a family is each way that the input declares how to read it. Accepted names the lines whose tests give an input that passes. Refused names the lines whose tests refuse an input, or `None`. Each refusal and each branch that classifies an input has a pair: a test that passes, and a test that refuses. Each line of Failures and of Acceptance is in a family. `just finish` fails when a family has no test. +A repair of a finding adds a pair of tests for its whole family, not only a test of the reported example. | Family | Source | Accepted | Refused | |---|---|---|---| @@ -71,7 +72,7 @@ Each line of Failures and of Acceptance is in a family. `just finish` fails when | `python3` runs the worker | Other: Missing program, and the intake | `size` | `no-python`, `worker-fails` | | A signal stops the run | Interruption | `size` | `interrupted`, `terminated` | -Fill: one row for each family of cases of your commands, in place of the rows of `inspect`. A repair of a finding adds a pair of tests for its whole family, not only a test of the reported example. +Fill: one row for each family of cases of your commands, in place of the rows of `inspect`. ## Open lines From 2e649bbef813b4e271d42846d6ce9fcff05e5c33 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:56:21 -0400 Subject: [PATCH 55/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20finish?= =?UTF-8?q?=20reads=20None=20in=20code=20in=20the=20matrix,=20and=20no=20g?= =?UTF-8?q?rade=20in=20the=20assertion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Refused cell of `None` in code, the form that the docs show, failed the matrix check. The check now strips the marks. The line Assertion of ## First failing run copies a line of the scenario, which can name a severity. Finish no longer reads it as a grade. Refs #422, #423 Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 1 + scripts/template-finish-matrix-test.sh | 5 +++-- scripts/template-ledger-run-test.sh | 3 +++ sdk/extensionrepo/readiness.go | 10 ++++++++-- sdk/extensionrepo/spec_matrix.go | 2 +- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index 963a2315..895bf8e2 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -35,6 +35,7 @@ After the first three lines, outside the line `- **Severity**:` of a finding, fi - A severity, then a hyphen and the word level, such as "warn-level". The text "block-level" fails too, so rephrase it - Under `## Findings`, the word severity alone +The line `- **Assertion**:` of `## First failing run` is exempt, because it copies a line of the scenario. Finish reads each other line as prose, and it passes. This is also true when the line grades in other words, such as "the grade is warn" or "a block for me". The reviewer gives each grade only in the bullet **Severity** of its finding, and moves each other grade there before the verdict. diff --git a/scripts/template-finish-matrix-test.sh b/scripts/template-finish-matrix-test.sh index 4c5a9d3a..9b565731 100644 --- a/scripts/template-finish-matrix-test.sh +++ b/scripts/template-finish-matrix-test.sh @@ -8,10 +8,11 @@ fct_test_finish_matrix() { local row='| The input holds text | Input: Of the wrong type | `lines` | `not-text` |' - fct_edit "${finish}/spec.md" "${row}" '| The input holds text | Input: Of the wrong type | `lines` | None |' + fct_edit "${finish}/spec.md" "${row}" '| The input holds text | Input: Of the wrong type | `lines` | `None` |' fct_finish 1 fct_finish_said 'the line `not-text` is in no family of ## Case matrix' - fct_edit "${finish}/spec.md" '| The input holds text | Input: Of the wrong type | `lines` | None |' '| The input holds text | Input: Of the wrong type | None | `not-text` |' + fct_finish_said 'names no spec ID of a refused input' absent + fct_edit "${finish}/spec.md" '| The input holds text | Input: Of the wrong type | `lines` | `None` |' '| The input holds text | Input: Of the wrong type | None | `not-text` |' fct_finish 1 fct_finish_said 'the family "The input holds text" names no spec ID of an input that passes' fct_edit "${finish}/spec.md" '| The input holds text | Input: Of the wrong type | None | `not-text` |' '| The input holds text | Input: Of the wrong type | `lines`, `utf-16` | `not-text` |' diff --git a/scripts/template-ledger-run-test.sh b/scripts/template-ledger-run-test.sh index 9d323bf6..dd9bb698 100644 --- a/scripts/template-ledger-run-test.sh +++ b/scripts/template-ledger-run-test.sh @@ -77,6 +77,9 @@ fct_test_ledger_run() { # A longer fence holds a shorter one, and a value can end with a period. ledger_run=$'````\n```\n````\n\n- **Commit**: '"$(fct_step "${finish}" scenario)"$'.\n- **Command:** `just scenario inspect`\n- **Exit**: 1.\n- **Assertion**: `stdout \'"lines":3\'` found no match.' fct_readiness "${head}" Ready "${valid}" 'agent B' fct_finish 0 + # The assertion copies a line of the scenario, so a severity in it grades nothing (#422). + ledger_run=$'- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `just scenario inspect`\n- **Exit**: 1\n- **Assertion**: `stdout \'"lines":3\'` found no match, not `warn`' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 0 # The command is just scenario , of a scenario of the Scenario commit, and the # assertion holds a line of that scenario. So signoff can run it again (#422). diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index 846b2613..afdb4c60 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -37,7 +37,7 @@ func readinessFindings(readiness string) ([]finding, []string) { const fix = "Fix: give each finding its own ### heading under ## Findings, with one line - **Severity**: `block`, `warn`, or `advise`" var findings []finding var problems []string - part, current, fence, opened, none, stray, seen := 0, -1, "", 0, false, false, false + part, current, fence, opened, none, stray, seen, run := 0, -1, "", 0, false, false, false, false // mentions holds the line of each finding ID outside ## Findings. mentions := map[string]int{} closeFindings := func() { @@ -52,9 +52,15 @@ func readinessFindings(readiness string) ([]finding, []string) { // A heading of level 1 or 2 closes ## Findings, so it is no line of a finding. section := fence == "" && heading != nil && len(heading[1]) <= 2 bullet := part > 0 && fence == "" && m != nil && m[1] == "Severity" + // The assertion of ## First failing run copies a line of the scenario, which can name a severity (#422). + if section { + run = heading[2] == "First failing run" + } + record := firstRun.FindStringSubmatch(text) + assertion := run && fence == "" && record != nil && record[1] == "Assertion" // The first three lines are the header. A severity elsewhere fails, // also under another part after ## Findings (#416). - if i >= 3 && !bullet && (readinessSeverity.MatchString(text) || severityCell(text) != "" || readinessGrade.MatchString(text) || readinessLevel.MatchString(text) || + if i >= 3 && !bullet && !assertion && (readinessSeverity.MatchString(text) || severityCell(text) != "" || readinessGrade.MatchString(text) || readinessLevel.MatchString(text) || part > 0 && !section && readinessWord.MatchString(text)) { problems = append(problems, fmt.Sprintf("readiness.md:%d: %s", i+1, severityCause(text, fence))) } diff --git a/sdk/extensionrepo/spec_matrix.go b/sdk/extensionrepo/spec_matrix.go index 489907ec..1a9a5027 100644 --- a/sdk/extensionrepo/spec_matrix.go +++ b/sdk/extensionrepo/spec_matrix.go @@ -48,7 +48,7 @@ func matrixProblems(spec string, limits []limit) []string { problems = append(problems, at+"names no spec ID of an input that passes. "+ "Fix: name in Accepted the line whose test gives an input that passes, so each refusal has its pair") } - if len(refused) == 0 && strings.TrimSuffix(l.Cells[3], ".") != "None" { + if len(refused) == 0 && strings.Trim(l.Cells[3], "`*_.") != "None" { problems = append(problems, at+"names no spec ID of a refused input, and not None. Fix: name in Refused the line whose test refuses an input, or write None") } for _, id := range append(accepted, refused...) { From 674406932998e571fb89bd53e30422e8c096c2e8 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 00:59:16 -0400 Subject: [PATCH 56/61] =?UTF-8?q?=F0=9F=90=9B=20fix:=20template:=20only=20?= =?UTF-8?q?the=20first=20assertion=20of=20the=20first=20failing=20run=20is?= =?UTF-8?q?=20exempt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each line Assertion under each ## First failing run was exempt from the severity forms, while finish read only the last one as the record. Only the first one is now exempt, and finish reads that one. A second line Assertion that grades fails finish. Refs #422 Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 2 +- scripts/template-ledger-run-test.sh | 4 ++++ sdk/extensionrepo/ledger_run.go | 5 ++++- sdk/extensionrepo/readiness.go | 6 ++++-- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index 895bf8e2..5452918f 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -35,7 +35,7 @@ After the first three lines, outside the line `- **Severity**:` of a finding, fi - A severity, then a hyphen and the word level, such as "warn-level". The text "block-level" fails too, so rephrase it - Under `## Findings`, the word severity alone -The line `- **Assertion**:` of `## First failing run` is exempt, because it copies a line of the scenario. +The first line `- **Assertion**:` of `## First failing run` is exempt, because it copies a line of the scenario. Finish reads only that line as the assertion. Finish reads each other line as prose, and it passes. This is also true when the line grades in other words, such as "the grade is warn" or "a block for me". The reviewer gives each grade only in the bullet **Severity** of its finding, and moves each other grade there before the verdict. diff --git a/scripts/template-ledger-run-test.sh b/scripts/template-ledger-run-test.sh index dd9bb698..a4daacfd 100644 --- a/scripts/template-ledger-run-test.sh +++ b/scripts/template-ledger-run-test.sh @@ -80,6 +80,10 @@ fct_test_ledger_run() { # The assertion copies a line of the scenario, so a severity in it grades nothing (#422). ledger_run=$'- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `just scenario inspect`\n- **Exit**: 1\n- **Assertion**: `stdout \'"lines":3\'` found no match, not `warn`' fct_readiness "${head}" Ready "${valid}" 'agent B' fct_finish 0 + # Only the first assertion is exempt, and it is the one that finish reads. + ledger_run=$'- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `just scenario inspect`\n- **Exit**: 1\n- **Assertion**: `stdout \'"lines":3\'` found no match\n- **Assertion**: Grade: block for the empty input' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said 'this line grades with the severity block' # The command is just scenario , of a scenario of the Scenario commit, and the # assertion holds a line of that scenario. So signoff can run it again (#422). diff --git a/sdk/extensionrepo/ledger_run.go b/sdk/extensionrepo/ledger_run.go index d55de53a..7437d1b9 100644 --- a/sdk/extensionrepo/ledger_run.go +++ b/sdk/extensionrepo/ledger_run.go @@ -113,8 +113,11 @@ func firstRunRecord(readiness string) (map[string]string, bool) { } continue } + // The first value of each line counts, as for the severity forms in readinessFindings. if m := firstRun.FindStringSubmatch(text); at > 0 && m != nil { - parts[m[1]] = strings.Trim(strings.TrimSuffix(m[2], "."), "`") + if _, read := parts[m[1]]; !read { + parts[m[1]] = strings.Trim(strings.TrimSuffix(m[2], "."), "`") + } } } return parts, found diff --git a/sdk/extensionrepo/readiness.go b/sdk/extensionrepo/readiness.go index afdb4c60..8569d9ed 100644 --- a/sdk/extensionrepo/readiness.go +++ b/sdk/extensionrepo/readiness.go @@ -37,7 +37,7 @@ func readinessFindings(readiness string) ([]finding, []string) { const fix = "Fix: give each finding its own ### heading under ## Findings, with one line - **Severity**: `block`, `warn`, or `advise`" var findings []finding var problems []string - part, current, fence, opened, none, stray, seen, run := 0, -1, "", 0, false, false, false, false + part, current, fence, opened, none, stray, seen, run, exempt := 0, -1, "", 0, false, false, false, false, true // mentions holds the line of each finding ID outside ## Findings. mentions := map[string]int{} closeFindings := func() { @@ -57,7 +57,9 @@ func readinessFindings(readiness string) ([]finding, []string) { run = heading[2] == "First failing run" } record := firstRun.FindStringSubmatch(text) - assertion := run && fence == "" && record != nil && record[1] == "Assertion" + // Only the first one is exempt, the one that firstRunRecord reads. + assertion := exempt && run && fence == "" && record != nil && record[1] == "Assertion" + exempt = exempt && !assertion // The first three lines are the header. A severity elsewhere fails, // also under another part after ## Findings (#416). if i >= 3 && !bullet && !assertion && (readinessSeverity.MatchString(text) || severityCell(text) != "" || readinessGrade.MatchString(text) || readinessLevel.MatchString(text) || From f9dbd7ec26ee7d61d93ce69c5a8fbc9d3f320088 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 01:01:39 -0400 Subject: [PATCH 57/61] =?UTF-8?q?=E2=9C=85=20test:=20template:=20finish=20?= =?UTF-8?q?reads=20the=20first=20assertion=20of=20the=20first=20failing=20?= =?UTF-8?q?run?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A record whose first line Assertion grades and holds no line of the scenario fails finish, even when a second line Assertion is valid. So the record parser keeps reading the first value, as the severity forms do. Refs #422 Co-Authored-By: Claude Opus 5.5 --- scripts/template-ledger-run-test.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/template-ledger-run-test.sh b/scripts/template-ledger-run-test.sh index a4daacfd..29b7691c 100644 --- a/scripts/template-ledger-run-test.sh +++ b/scripts/template-ledger-run-test.sh @@ -84,6 +84,9 @@ fct_test_ledger_run() { ledger_run=$'- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `just scenario inspect`\n- **Exit**: 1\n- **Assertion**: `stdout \'"lines":3\'` found no match\n- **Assertion**: Grade: block for the empty input' fct_readiness "${head}" Ready "${valid}" 'agent B' fct_finish 1 fct_finish_said 'this line grades with the severity block' + ledger_run=$'- **Commit**: '"$(fct_step "${finish}" scenario)"$'\n- **Command**: `just scenario inspect`\n- **Exit**: 1\n- **Assertion**: Grade: block for the empty input\n- **Assertion**: `stdout \'"lines":3\'` found no match' fct_readiness "${head}" Ready "${valid}" 'agent B' + fct_finish 1 + fct_finish_said 'names the assertion "Grade: block for the empty input", which holds no line' # The command is just scenario , of a scenario of the Scenario commit, and the # assertion holds a line of that scenario. So signoff can run it again (#422). From 97ddaf756fa2d71c4e89f590da3b09fb60b6d1f8 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 01:02:56 -0400 Subject: [PATCH 58/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20verify-nicely:=20t?= =?UTF-8?q?he=20loop=20text=20of=20the=20template=20names=20its=20case=20m?= =?UTF-8?q?atrix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The step Loop text of the feature template now checks that the rendered spec.md holds ## Case matrix beside ## Open lines. Refs #423 Co-Authored-By: Claude Opus 5.5 --- .agents/skills/verify-nicely/features/template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agents/skills/verify-nicely/features/template.md b/.agents/skills/verify-nicely/features/template.md index 774f346f..1be59c8e 100644 --- a/.agents/skills/verify-nicely/features/template.md +++ b/.agents/skills/verify-nicely/features/template.md @@ -42,7 +42,7 @@ Preconditions: - **Remove command.** In the output of `skill view nicely-extensions`, the skill holds the section `## Remove an extension`. It names `ncly doctor extensions --json`, the four files `ncly-SLUG`, `ncly-SLUG.toml`, `ncly-SLUG.locales/`, and `ncly-SLUG.skill/`, `just uninstall`, `NOT_FOUND`, and `GRANT_UNUSED`. - **Render and install.** Run `just verify prepare "$run" template`. It exits 0. Its log names `just install` of `probe`, the line `installed /scratch/home/.local/share/nicely/extensions/ncly-probe`, and the `python3` that it puts first in the PATH of the wrapper. Then run `ls "$run/scratch/home/.local/share/nicely/extensions"`: it lists `ncly-probe`, `ncly-probe.locales`, `ncly-probe.skill`, and `ncly-probe.toml`, beside the four files of `hello`. Report the output of `prepare` and of `ls` beside the quote. - **Release and install text.** Run `grep -n -A12 '^## Release$' "$run/scratch/ncly-probe/AGENTS.md"` and `grep -n -A3 '^git clone' "$run/scratch/ncly-probe/README.md"`. Both exit 0. `AGENTS.md` names `just release vX.Y.Z` and the release receipt. `README.md` holds the lines `git clone --branch TAG https://example.com/ncly-probe ncly-probe`, `cd ncly-probe`, `just install`, and `ncly describe probe --json`. Report both outputs beside the quote. -- **Loop text.** Run `grep -n -A12 '^## Done when$' "$run/scratch/ncly-probe/AGENTS.md"`, then `grep -n '^## Open lines$' "$run/scratch/ncly-probe/spec.md"`. Both exit 0. "Done when" names `just finish`, the hosted checks, 5 rounds, and a "full stop" request that runs only after it. Then run `grep -n '^##' "$run/scratch/ncly-probe/.agents/skills/nicely-extension/SKILL.md"`. It lists `## Intake`, `## Acceptance lines`, `## Test files`, `## Rounds`, and `## Readiness review`. Then run `grep -n '^##' "$run/scratch/ncly-probe/.agents/skills/nicely-extension/intake.md"`. It lists `## Intake of a script`, `## Intake of a program`, and `## Evidence of the intake`. Report the four outputs beside the quote. +- **Loop text.** Run `grep -n -A12 '^## Done when$' "$run/scratch/ncly-probe/AGENTS.md"`, then `grep -n '^## Open lines$\|^## Case matrix$' "$run/scratch/ncly-probe/spec.md"`. Both exit 0, and `spec.md` lists both parts. "Done when" names `just finish`, the hosted checks, 5 rounds, and a "full stop" request that runs only after it. Then run `grep -n '^##' "$run/scratch/ncly-probe/.agents/skills/nicely-extension/SKILL.md"`. It lists `## Intake`, `## Acceptance lines`, `## Test files`, `## Rounds`, and `## Readiness review`. Then run `grep -n '^##' "$run/scratch/ncly-probe/.agents/skills/nicely-extension/intake.md"`. It lists `## Intake of a script`, `## Intake of a program`, and `## Evidence of the intake`. Report the four outputs beside the quote. - **Finish gate.** Run `(cd "$run/scratch/ncly-probe" && just finish)`, whose subshell keeps the folder of the caller. Exit 1. Stderr names each `Fill:` line of `vision.md`, `spec.md`, and `qa.md` as `:: this line of the form is not filled`, and `readiness.md: no readiness verdict is recorded`. Report the output beside the quote. - **Discovery.** Run `just verify step "$run" template --help`. Exit 0, and the `EXTENSIONS` section lists `probe: Inspect files` beside `hello` and `skill`. - **Help.** Run `just verify step "$run" template probe --help`. Exit 0, and `COMMANDS` lists `inspect: Give the size and the line count of a file`. From a3b4730f30c5a19d2fce54672c95f743f8a74b02 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 12:08:35 -0500 Subject: [PATCH 59/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20template:=20the=20?= =?UTF-8?q?case=20matrix=20check=20reads=20the=20references=20only?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The check of ## Case matrix proves that each family names a test and that each line of Failures and Acceptance has a family. It cannot prove that the matrix is complete or correct. Say so in finish-gate.md, the skill, the readiness review, and the rule index (judge 6 of the day-1 trials). Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 2 ++ template/.agents/skills/nicely-extension/SKILL.md | 1 + template/.agents/skills/nicely-extension/readiness-review.md | 1 + template/AGENTS/rule-index.md.jinja | 2 +- 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index 5452918f..1fd0eac9 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -19,6 +19,8 @@ This part says what `just finish` checks in an extension repository, as [Extensi - `Ready` with a `block` or `warn` finding that is not `fixed`, or with a limit in `spec.md` - `Ready with stated limits` with a `block` finding that is not `fixed`, or with a `warn` that is neither `fixed` nor an accepted `limit` +The check of `## Case matrix` reads the references only. It cannot prove that the matrix is complete or correct. + Finish reads a severity only in the line `- **Severity**:` of a finding. The plain words block, warn, and advise pass in a title, in evidence, and in prose (#419). An example is the title `### R1-F1 Large inputs block the terminal`. diff --git a/template/.agents/skills/nicely-extension/SKILL.md b/template/.agents/skills/nicely-extension/SKILL.md index 61121a7d..7e819db4 100644 --- a/template/.agents/skills/nicely-extension/SKILL.md +++ b/template/.agents/skills/nicely-extension/SKILL.md @@ -90,6 +90,7 @@ Before the review of round 1, `## Case matrix` of `spec.md` lists each family of - A repair of a finding adds a pair of tests for its whole family, not only a test of the reported example `just finish` fails when a family names no test, or when a line of Failures or Acceptance is in no family. +It reads the references only. The first reviewer checks that the matrix is complete. ### Each next round diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index b81e0718..0932c7d0 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -91,6 +91,7 @@ First, before you review any repair, check `## Case matrix` of `spec.md` (#423): - Each generic case that applies, and each family that the intake found, has its row - Each refusal and each branch that classifies an input has a pair of tests: one input passes, and one input is refused - Run the tests of the matrix. A family that the matrix lacks is a finding +- `just finish` checks only that each family names a test. It does not check that the matrix is complete, or that each test proves its family Then review the rest of round 1: diff --git a/template/AGENTS/rule-index.md.jinja b/template/AGENTS/rule-index.md.jinja index d92ee1aa..58470d45 100644 --- a/template/AGENTS/rule-index.md.jinja +++ b/template/AGENTS/rule-index.md.jinja @@ -37,7 +37,7 @@ The paragraphs of `AGENTS.md` state the rules marked "Prose only". | Each case of the Output group that applies has a test file that the tests make, checked by an independent reader | Prose only | | The tests make their files; never search the home folder or copy a file of the user | Prose only | | Round 1 tests each generic case that applies, and the first review tests families of cases | Prose only | -| Before the review of round 1, `## Case matrix` gives each family its tests, and each line of Failures and Acceptance a family | [Finish matrix][finish-matrix] | +| Before the review of round 1, `## Case matrix` gives each family its tests, and each line of Failures and Acceptance a family | [Finish matrix][finish-matrix] checks the references only, not that the matrix is complete or correct | | Each refusal and each branch that classifies an input has a pair of tests | [Finish matrix][finish-matrix] needs a spec ID in Accepted of each family; the reviewer checks that it passes | | The first reviewer checks that the matrix is complete and runs it, before any repair | Prose only | | A repair adds a pair of tests for its whole family, and only then is its finding `fixed` | Prose only | From 1a4556c803cc4da96e3196271a97539d2c8966d8 Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 20:25:35 -0500 Subject: [PATCH 60/61] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor:=20docs:=20?= =?UTF-8?q?move=20the=20limit=20of=20the=20parser=20to=20its=20own=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Sol fix of #421 added a sentence to finish-gate.md, and with the case matrix of #424 the file went over the 2500 tokens of just limits. The part "The limit of the parser" now has its own file, with no change of its text. finish-gate.md and readiness-review.md link it. Refs #423 Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 18 ++---------------- .../finish-gate/limit-of-the-parser.md | 17 +++++++++++++++++ .../nicely-extension/readiness-review.md | 2 +- 3 files changed, 20 insertions(+), 17 deletions(-) create mode 100644 docs/north-star/contract/finish-gate/limit-of-the-parser.md diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index 1fd0eac9..fa0d3d34 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -14,7 +14,7 @@ This part says what `just finish` checks in an extension repository, as [Extensi - A `readiness.md` whose first three lines hold no `Commit: `, no `Reviewer: `, or no verdict `Ready` or `Ready with stated limits`. A byte order mark before them is ignored - A reviewed commit that differs from the working tree outside `readiness.md`, and outside the part **Limits.** of `spec.md`. So the user accepts a limit after a review, with no new round - A finding of `readiness.md` without its own `###` heading under `## Findings`, or without one severity: `block`, `warn`, or `advise` -- A severity after the first three lines, outside the line `- **Severity**:` of a finding, in a form that grades. [The limit of the parser](#the-limit-of-the-parser) lists each such form. The message names the word or the table that caused it +- A severity after the first three lines, outside the line `- **Severity**:` of a finding, in a form that grades. [The limit of the parser](finish-gate/limit-of-the-parser.md) lists each such form. The message names the word or the table that caused it - A finding heading with an ID, such as `### R1-F2`, outside `## Findings`. Also a line outside `## Findings` that names the ID of no finding under it - `Ready` with a `block` or `warn` finding that is not `fixed`, or with a limit in `spec.md` - `Ready with stated limits` with a `block` finding that is not `fixed`, or with a `warn` that is neither `fixed` nor an accepted `limit` @@ -25,21 +25,7 @@ Finish reads a severity only in the line `- **Severity**:` of a finding. The plain words block, warn, and advise pass in a title, in evidence, and in prose (#419). An example is the title `### R1-F1 Large inputs block the terminal`. -### The limit of the parser - -The severities are the words block, warn, and advise, in any case. -After the first three lines, outside the line `- **Severity**:` of a finding, finish fails on these forms, and only on these (#413, #416, #420, #422): - -- A severity in code, such as `` `warn` ``, also in fenced code -- A cell of a table that holds only a severity, with only marks around it, such as `` `warn` ``, `**block**`, or `(advise)` -- In a row that names a finding ID, a cell whose last word is a severity, such as `| R1-F1 | really a warn |` -- The word severity, grade, grades, or graded, then only marks or spaces, then a severity, such as "Severity: block" or "Grade: warn" -- A severity, then a hyphen and the word level, such as "warn-level". The text "block-level" fails too, so rephrase it -- Under `## Findings`, the word severity alone - -The first line `- **Assertion**:` of `## First failing run` is exempt, because it copies a line of the scenario. Finish reads only that line as the assertion. -Finish reads each other line as prose, and it passes. This is also true when the line grades in other words, such as "the grade is warn" or "a block for me". -The reviewer gives each grade only in the bullet **Severity** of its finding, and moves each other grade there before the verdict. +[The limit of the parser](finish-gate/limit-of-the-parser.md) lists the forms of a severity that finish fails on. ## The ledger diff --git a/docs/north-star/contract/finish-gate/limit-of-the-parser.md b/docs/north-star/contract/finish-gate/limit-of-the-parser.md new file mode 100644 index 00000000..65a60915 --- /dev/null +++ b/docs/north-star/contract/finish-gate/limit-of-the-parser.md @@ -0,0 +1,17 @@ +# The limit of the parser + +This part of [Finish gate](../finish-gate.md) lists the forms of a severity that `just finish` fails on. + +The severities are the words block, warn, and advise, in any case. +After the first three lines, outside the line `- **Severity**:` of a finding, finish fails on these forms, and only on these (#413, #416, #420, #422): + +- A severity in code, such as `` `warn` ``, also in fenced code +- A cell of a table that holds only a severity, with only marks around it, such as `` `warn` ``, `**block**`, or `(advise)` +- In a row that names a finding ID, a cell whose last word is a severity, such as `| R1-F1 | really a warn |` +- The word severity, grade, grades, or graded, then only marks or spaces, then a severity, such as "Severity: block" or "Grade: warn" +- A severity, then a hyphen and the word level, such as "warn-level". The text "block-level" fails too, so rephrase it +- Under `## Findings`, the word severity alone + +The first line `- **Assertion**:` of `## First failing run` is exempt, because it copies a line of the scenario. Finish reads only that line as the assertion. +Finish reads each other line as prose, and it passes. This is also true when the line grades in other words, such as "the grade is warn" or "a block for me". +The reviewer gives each grade only in the bullet **Severity** of its finding, and moves each other grade there before the verdict. diff --git a/template/.agents/skills/nicely-extension/readiness-review.md b/template/.agents/skills/nicely-extension/readiness-review.md index 0932c7d0..0bf38030 100644 --- a/template/.agents/skills/nicely-extension/readiness-review.md +++ b/template/.agents/skills/nicely-extension/readiness-review.md @@ -48,7 +48,7 @@ After the first three lines, it fails on each form that grades in another place: - The word severity under `## Findings` The plain words pass in a title, in evidence, and in prose, such as `### R1-F1 Large inputs block the terminal`. -`just finish` does not find a grade in other words, such as "a block for me". "The limit of the parser" in `finish-gate.md` gives the exact forms. +`just finish` does not find a grade in other words, such as "a block for me". "The limit of the parser" of the finish gate of Nicely, `docs/north-star/contract/finish-gate/limit-of-the-parser.md`, gives the exact forms. Before the verdict, read the whole review, also the notes. Move each grade in another form to the bullet **Severity** of its finding. A loss that a user can see is `warn` or `block`, never `advise`. From 19ff6be772504a543d6c6af7dabd41530589d9cf Mon Sep 17 00:00:00 2001 From: Pascal Andy Date: Sun, 1 Nov 2026 20:25:45 -0500 Subject: [PATCH 61/61] =?UTF-8?q?=F0=9F=93=9D=20docs:=20finish=20gate:=20a?= =?UTF-8?q?n=20accepted=20open=20finding=20leaves=20##=20Open=20lines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The list of what finish fails on now says that a row of ## Open lines may leave when its ID became a row of **Limits.**, as e114cd1d allows. Refs #420 Co-Authored-By: Claude Opus 5.5 --- docs/north-star/contract/finish-gate.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/north-star/contract/finish-gate.md b/docs/north-star/contract/finish-gate.md index fa0d3d34..c89eb478 100644 --- a/docs/north-star/contract/finish-gate.md +++ b/docs/north-star/contract/finish-gate.md @@ -12,7 +12,7 @@ This part says what `just finish` checks in an extension repository, as [Extensi - No table under `## Case matrix` of `spec.md` with the columns Family, Source, Accepted, and Refused (#423). Also a family that names no spec ID in Accepted, or a Refused cell with no spec ID and not `None` - A spec ID of `## Case matrix` that is no line of Failures or Acceptance and no accepted limit, so no test proves it. Also a line of Failures or Acceptance that no family names - A `readiness.md` whose first three lines hold no `Commit: `, no `Reviewer: `, or no verdict `Ready` or `Ready with stated limits`. A byte order mark before them is ignored -- A reviewed commit that differs from the working tree outside `readiness.md`, and outside the part **Limits.** of `spec.md`. So the user accepts a limit after a review, with no new round +- A reviewed commit that differs from the working tree outside `readiness.md`, and outside the part **Limits.** of `spec.md`. So the user accepts a limit after a review, with no new round. A row of `## Open lines` whose ID became a row of **Limits.** may leave too - A finding of `readiness.md` without its own `###` heading under `## Findings`, or without one severity: `block`, `warn`, or `advise` - A severity after the first three lines, outside the line `- **Severity**:` of a finding, in a form that grades. [The limit of the parser](finish-gate/limit-of-the-parser.md) lists each such form. The message names the word or the table that caused it - A finding heading with an ID, such as `### R1-F2`, outside `## Findings`. Also a line outside `## Findings` that names the ID of no finding under it