diff --git a/.claude/skills/code-comments/SKILL.md b/.claude/skills/code-comments/SKILL.md new file mode 100644 index 000000000..9bc0653a6 --- /dev/null +++ b/.claude/skills/code-comments/SKILL.md @@ -0,0 +1,75 @@ +--- +name: code-comments +description: MUST use before writing or editing any comment in this repository - content rules, budget, width. +--- + +# Machine code comments + +Before you write a comment, delete it. If the code and its names already say it, +it was noise. What survives explains a contract, an invariant, a compatibility +requirement, a performance tradeoff, or a non-obvious reason. + +Say WHAT the code guarantees and WHY, in the present tense. Do not narrate HOW +it works - the comment should survive an equivalent rewrite. A member summary +describes that member's own contract, not its caller's. + +Match the file you are editing. Placement, density, and idiom are local here; +read what is already there before you add to it, and use the terms it uses. + +## Never write these + +`scripts/comment-hygiene.ps1` fails on them over the lines your branch adds: + +1. **Process framing** - `Phase 1`, `later we'll`, `we'll eventually`. +2. **History** - `it used to`, `previously returned`, `was removed`, + `renamed from`, `no longer used`. +3. **Provenance** - `extracted from`, `shared by X and Y`, `the only caller`. +4. **Pointers** - to a Markdown file, a numbered section, a review note, or + another file's comment. +5. **Non-ASCII punctuation** - use `--`, `->`, `...`, `-`, `x`, plain quotes. + Typography only; comment text may use any script the language data needs. + +Present tense about current state is not history: "Returns null when the stratum +has no rules" is a contract. A compatibility note about behavior that must stay +true is welcome, as is an issue reference that is part of the contract. + +## Fit the budget + +One block - a run of whole-line `//` or `#` comments, ended by a blank line, +code, or a doc comment - gets **200 characters total**, markers and indentation +excluded. Every line fits **120 display columns**. + +`///` blocks and PowerShell block comments are exempt from the budget, not from +the content rules or the width limit. + +Over budget? Shorten it, or let the code express it directly. Do not convert a +`//` block to `///` to buy the exemption. + +The tree already holds comments over budget. They are known debt, not a +convention: do not copy them, and do not sweep them either. Shorten one when you +are already changing the code it describes. + +## XML documentation + +Prefer none. An undocumented public type is the norm here, several projects use +`///` nowhere at all, and HermitCrab's heavier use is not a model to copy. +`SIL.Machine.Morphology.HermitCrab`, `SIL.Machine.Tokenization.SentencePiece`, +and `SIL.Machine.Translation.TensorFlow` do not set `GenerateDocumentationFile` +at all, so a `///` block there ships nothing to a consumer. + +Write one only when a caller needs a contract the signature cannot state: units, +nullability, ownership, an exception they must handle. Then one `` +above the member. Omit `` and `` that only restate a name or +type; keep them for real result semantics. Document every parameter or none. No +file headers, no divider comments, no fact repeated in both the summary and the +parameters. + +A test comment explains a non-obvious fixture or setup constraint. It does not +restate the test name. + +## Run the check + +`pwsh ./scripts/comment-hygiene.ps1` scans the lines your branch adds. Add +`-Full -Advisory` to size existing debt, or `-SelfTest` to check the rules +themselves. Agents run `./local_check.sh --agent-strict`, which makes the scan +blocking; the pull request check is advisory, so its green tick proves nothing. diff --git a/.claude/skills/commit-messages/SKILL.md b/.claude/skills/commit-messages/SKILL.md new file mode 100644 index 000000000..ea25a4d1a --- /dev/null +++ b/.claude/skills/commit-messages/SKILL.md @@ -0,0 +1,26 @@ +--- +name: commit-messages +description: How to write a commit message in sillsdev/machine - imperative subject, short body. +--- + +# Commit messages + +Name the change in an imperative, sentence-case subject under about 72 +characters, with no terminal punctuation: + +- `Fix bug in MergeEquivalentAnalyses (#493)` +- `Port changes from sillsdev/machine.py#336 (#498)` + +A body, when there is one: blank line after the subject, wrapped at about 80 +columns, saying what changed and why. Reference a GitHub issue when one exists. + +## Two traps in the history + +1. The `(#N)` suffix is added by GitHub when a pull request is squashed. Never + type it into a local commit. +2. Older commits carry Jira identifiers such as `LT-22605`. That is historical; + use a GitHub issue reference. + +The 72-character limit is not enforced and longer subjects exist. Do not rewrite +shared history to satisfy it, or to fix a message on a pushed branch - add a +corrective commit unless the author asks for the rewrite. diff --git a/.claude/skills/issue-authoring/SKILL.md b/.claude/skills/issue-authoring/SKILL.md new file mode 100644 index 000000000..dd38791ed --- /dev/null +++ b/.claude/skills/issue-authoring/SKILL.md @@ -0,0 +1,58 @@ +--- +name: issue-authoring +description: How to write a GitHub issue in sillsdev/machine - one symptom, short body, real evidence. +argument-hint: Optional issue type, title, symptoms, acceptance criteria, or source PR +user-invocable: true +--- + +# Writing a machine issue + +Search open and recently closed issues first, and say what you searched. Then +write the title and the three-sentence lede; the form fields hold the rest. + +GitHub issues are the tracker here. An `LT-` reference is an external link, and +only when someone supplied it. + +## 1. Title: one symptom + +Under about 70 characters, in the reader's words. No "investigate", no +"improve", no component prefix the labels already carry. + +Bad: *Tokenizer improvements* +Good: *USFM attribute is dropped when the locale is tr-TR* + +## 2. Lede: three sentences + +1. **The symptom** - what goes wrong. +2. **The trigger** - the smallest condition that produces it. +3. **The cost** - who is blocked, or what the caller sees instead. + +Under 25 words each. For a feature, the same three: what is missing, when it +bites, what it costs. + +Good: *A USFM attribute is dropped when the tokenizer runs under tr-TR. Any +marker containing an ASCII `i` splits at the wrong index on a Turkish locale. +Round-tripping a Turkish project silently loses the attribute.* + +## 3. Body: labelled lines, never a wall + +Write `Unknown` where you do not know, and say how to find out. + +- **Bug** - affected API; version, OS, runtime; the smallest input that shows + it; expected vs actual; sanitized log; when it started; the test that catches + it. +- **Feature** - who is blocked and by what; the proposed behavior and its + compatibility cost; acceptance criteria an outsider could check; non-goals. +- **Porting** - the source PR URL, what behavior matters here, what does not. + +Sanitize first: no secrets, tokens, customer text, or private project data. + +## 4. Check it is ready + +Ready means another maintainer can reproduce the bug, judge the acceptance +criteria, or find the change to port - without asking you a question. + +A workflow files the porting issue after a merge, marked `AUTO-GENERATED-ISSUE`. +Do not write a second one by hand. + +Hand back the title, labels, and body. The author decides whether to publish. diff --git a/.claude/skills/pr-authoring/SKILL.md b/.claude/skills/pr-authoring/SKILL.md new file mode 100644 index 000000000..98c325774 --- /dev/null +++ b/.claude/skills/pr-authoring/SKILL.md @@ -0,0 +1,86 @@ +--- +name: pr-authoring +description: How to write a pull request in sillsdev/machine - strong lede, short body, honest evidence. +argument-hint: Optional branch purpose, issue number, or PR number +user-invocable: true +--- + +# Writing a machine PR + +Write the body to a file, then `gh pr edit --body-file`. Do not push, open, +or edit a PR unless the author asked. + +What to check and what to run is in `AGENTS.md` and `docs/review/`. This is the +write-up. + +## 1. Write the lede + +Three sentences, each with a different job, before anything else: + +1. **What it does** - what a caller can now do, or what stopped being broken. +2. **The first unknown, answered** - usually "what breaks?" or "why so big?" +3. **The boundary** - what it does not touch. + +Under 25 words each. A sentence needing a subordinate clause belongs in the +body. + +Bad: *This PR refactors the tokenizer and adds some tests.* + +Good: *USFM markers now split identically under tr-TR, where the attribute used +to be dropped. No public signature changes - the fix is one comparison, from +culture-aware to ordinal. Nothing outside `UsfmTokenizer` is touched.* + +Invisible to callers? Lead with what it protects: *Agents can no longer land a +comment that narrates its own history.* + +## 2. Fill the top zone + +Under 200 words. Drop any section that would be empty. + +```markdown +## Quick summary + + +## Where to look +- -- + +## Deliberately not included +- + +## Validation +- -- + +## Issue / porting context + +``` + +This mirrors `.github/PULL_REQUEST_TEMPLATE.md`. If the two ever differ, the +template is what contributors actually see; fix this to match it. + +## 3. Put the reasoning below the rule + +Everything longer goes under a `---`, in closed `
` blocks: *Reading +this a year from now*, *Decisions, and why*, *Paths not taken*, *Deferred, and +what would unblock it*. Long reasoning is welcome there and nowhere above. + +No preamble, no apology, no "should be fine", no recap. + +## 4. Check the claims + +Every count, path, type, and test name must match the tree. A wrong number in a +PR body outlives the PR. + +Validation lines carry the command and its result, nothing else. Never list a +command you did not run. Never call a local run CI-equivalent - CI collects +coverage and `local_check.sh` does not. Name any check you skipped. + +## Replying to review comments + +Reply in the thread, on the line, in two or three sentences. Classify first: + +- **Fix** - sound and unambiguous; make the smallest change. +- **Clarify** - ask the one specific question. +- **Reply only** - state the verified behavior; change nothing. +- **Defer** - name the follow-up and why it is outside this PR. + +Resolve only a thread that is fully answered and that you did not dispute. diff --git a/.claude/skills/pr-review/SKILL.md b/.claude/skills/pr-review/SKILL.md new file mode 100644 index 000000000..3701b384c --- /dev/null +++ b/.claude/skills/pr-review/SKILL.md @@ -0,0 +1,70 @@ +--- +name: pr-review +description: How to write up a code review in sillsdev/machine - short line comments, evidence, severity. +argument-hint: "[optional PR number, branch, or review focus]" +user-invocable: true +--- + +# Writing a machine review + +Post one short comment per finding, anchored on the line it is about, then one +summary comment. A review is read-only: do not edit, commit, push, or resolve +threads. + +What to look for is in `docs/review/`; `AGENTS.md` maps a changed path to its +rules file. + +## 1. One finding, one comment + +Anchor it on the line. Two problems on one line are two comments. A reviewer +scrolling the diff should meet each point where it applies. + +## 2. Lead with the claim + +First sentence names the defect. Evidence second, fix third, if it fits. + +``` +Ordinal comparison missing: `marker.IndexOf(":")` is culture-sensitive, so +tr-TR splits this marker differently. Pass `StringComparison.Ordinal`. +``` + +Three lines is long. A finding needing more is a design question - raise it in +the summary instead. + +## 3. Label the severity + +**Critical** blocks merge, then **Important**, then **Minor**. Critical means +demonstrated: a failing command, a broken contract, a missing gate. A worry is +not Critical. + +## 4. Carry the evidence + +Every comment gets a `path:line` and a consequence. Mark what you did not +confirm `Unverified`; an unverified concern never blocks a merge. + +- Do not report pre-existing issues the diff does not touch. +- Do not ask for a migration, modernization, or benchmark the diff gave no + reason for. +- A search that found nothing proves absence only if you state what you + searched. +- Name the commands you ran and what they returned. `./local_check.sh` is the + full local sequence; an agent-authored branch also needs `--agent-strict`, and + a green advisory `Comment hygiene` check does not stand in for it. +- A coverage percentage is not evidence that a changed line is tested. + +## 5. Close with five lines + +1. Verdict: approve, approve with fixes, or request changes. +2. The one thing that matters most, with its `path:line`. +3. Counts by severity. +4. What you ran, and its result. +5. What you could not verify. + +Say which public API, target framework, package, or parity contract changed, or +`None verified`. + +Then mark each finding **changed**, **accepted**, or **unverified**. Of 140 +review threads here in three years, 128 have no follow-up, so nobody can tell +which findings mattered. Leave nothing implicit. + +For an adversarial second pass, apply `docs/review/devils-advocate.md`. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..83c3f06fe --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,68 @@ +name: Bug report +description: Report a reproducible problem in SIL Machine +title: "[Bug]: " +labels: + - bug +body: + - type: markdown + attributes: + value: | + Please remove secrets and private project data from examples and logs. + - type: textarea + id: summary + attributes: + label: Summary + description: >- + Three short sentences: the symptom, the smallest trigger, and what it + costs. Detail goes in the fields below. + placeholder: | + A USFM attribute is dropped when the tokenizer runs under tr-TR. + Any marker containing an ASCII i splits at the wrong index on a Turkish locale. + Round-tripping a Turkish project silently loses the attribute. + validations: + required: true + - type: input + id: package + attributes: + label: Affected package or API + description: Name the SIL.Machine package, project, or public API. + validations: + required: true + - type: input + id: version + attributes: + label: Version or commit + description: Include the package version or commit SHA. + validations: + required: true + - type: input + id: environment + attributes: + label: Environment + description: OS, architecture, and .NET runtime. + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Reproduction + description: Minimal input/fixture and exact steps, including frequency. + placeholder: | + 1. ... + 2. ... + Expected: ... + Actual: ... + validations: + required: true + - type: textarea + id: logs + attributes: + label: Logs or exception + description: Paste sanitized output, or write None. + - type: textarea + id: regression + attributes: + label: Regression and test idea + description: State the first known good version, if known, and the smallest regression test. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..0086358db --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..f84b15e09 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,53 @@ +name: Feature request +description: Propose a behavior or API improvement for SIL Machine +title: "[Feature]: " +labels: + - enhancement +body: + - type: textarea + id: summary + attributes: + label: Summary + description: >- + Three short sentences: what is missing, when it bites, and what it + costs. Detail goes in the fields below. + validations: + required: true + - type: textarea + id: problem + attributes: + label: Problem and users + description: Who needs this and what cannot they do today? + validations: + required: true + - type: textarea + id: use_cases + attributes: + label: Use cases + description: Give concrete workflows and representative inputs. + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposed behavior or API + description: Describe observable behavior; include compatibility concerns. + validations: + required: true + - type: textarea + id: non_goals + attributes: + label: Non-goals and deferred work + description: State what this request does not include. + - type: textarea + id: acceptance + attributes: + label: Acceptance criteria and tests + description: List observable outcomes, edge cases, and the test strategy. + validations: + required: true + - type: textarea + id: constraints + attributes: + label: Compatibility, performance, and platform constraints + description: Include package/API, resource, OS, or runtime constraints. diff --git a/.github/ISSUE_TEMPLATE/porting_request.yml b/.github/ISSUE_TEMPLATE/porting_request.yml new file mode 100644 index 000000000..f09b8a45b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/porting_request.yml @@ -0,0 +1,47 @@ +name: Port from machine.py +description: Track relevant behavior to port from sillsdev/machine.py +title: "Port: " +labels: + - porting +body: + - type: markdown + attributes: + value: | + A merged PR normally creates this issue automatically. Use this form when no generated issue exists. + - type: textarea + id: summary + attributes: + label: Summary + description: >- + Three short sentences: the behavior to port, why it matters here, and + what is out of scope. Detail goes in the fields below. + validations: + required: true + - type: input + id: source_pr + attributes: + label: machine.py source PR or commit + description: Full GitHub URL. + validations: + required: true + - type: textarea + id: behavior + attributes: + label: Behavior to port + description: Describe the relevant behavior and what is not relevant here. + validations: + required: true + - type: textarea + id: target + attributes: + label: Target projects and compatibility + description: Name verified target projects/APIs, platform implications, and compatibility risks. + validations: + required: true + - type: textarea + id: tests + attributes: + label: Evidence and tests + description: Link source validation and propose target regression/acceptance tests. Mark unknowns explicitly. + validations: + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 000000000..fa0dbd9c7 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,32 @@ +## Quick summary + + + +## Where to look + + + +## Deliberately not included + + + +## Validation + + + +- `./local_check.sh` -- +- `./local_check.sh --agent-strict` -- +- `git diff --check ...HEAD` -- +- + +## Issue / porting context + + + + diff --git a/.github/workflows/comment-hygiene.yml b/.github/workflows/comment-hygiene.yml new file mode 100644 index 000000000..01e8cfdd9 --- /dev/null +++ b/.github/workflows/comment-hygiene.yml @@ -0,0 +1,46 @@ +name: Comment hygiene + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + # Separate from CI Build so the report does not wait on the native build, the + # Release build, and the test matrix. + comment_hygiene: + name: Report comment hygiene (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-22.04, windows-latest] + + steps: + - name: Check out full history + uses: actions/checkout@v6 + with: + # The scan diffs against the merge base; a shallow checkout leaves + # that commit unreachable. + fetch-depth: 0 + + - name: Verify the hygiene rules + shell: pwsh + run: ./scripts/comment-hygiene.ps1 -SelfTest + + # Advisory: the scan annotates the lines this pull request adds and always + # exits 0. Agents get the blocking version through ./local_check.sh + # --agent-strict, so existing comment debt and any false positive cannot + # block a human contributor. + - name: Scan the lines this pull request adds + shell: pwsh + run: > + ./scripts/comment-hygiene.ps1 + -Advisory + -BaseRef '${{ github.event.pull_request.base.sha }}' diff --git a/.gitignore b/.gitignore index af6b4a93c..0437b6304 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ tests/SIL.Machine.Tests/Corpora/TestData/usfm/target/* tests/SIL.Machine.Tests/Corpora/TestData/project/* tests/SIL.Machine.Tests/Corpora/TestData/pretranslations.json .idea + +# Local, generated review summaries +.review/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..6e2e7194d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,80 @@ +# machine contributor and agent guide + +What an agent must do here, and where looking at the tree will mislead you. +Everything else - layout, target frameworks, what a workflow runs - read from +the tree; it is accurate and this file would only rot. + +If this file disagrees with `local_check.sh`, `.github/workflows/ci.yml`, or the +code you are changing, prefer the executable behavior and say so. + +## Validation + +Run `./local_check.sh` from the repository root: it restores, checks CSharpier +formatting, builds Release, and tests. Do not skip a failing step or report +success without fresh output. + +Agents must also run `./local_check.sh --agent-strict`, which makes +`scripts/comment-hygiene.ps1` blocking over the lines the branch adds. The +standard it enforces is `.claude/skills/code-comments/SKILL.md`. Do not drop the +flag to get a run through. + +CI collects coverage and `local_check.sh` does not, so a local run is never +coverage-equivalent. + +## Where the tree misleads + +- `ci.yml` triggers on `push`, not `pull_request`. A green check on a pull + request reflects the pushed head, not the merge result, and not a PR gate. +- The `Comment hygiene` check is advisory and never fails. Its green tick is not + evidence that the strict scan passed. +- `appveyor.yml` is legacy and names projects that do not exist. Ignore it; do + not repair it. +- A directory under `src/` or `tests/` is not an active project unless a current + project file, solution entry, or CI step references it. +- Three different things are called `Word` here: a corpus word position, + `WordAnalysis` (`src/SIL.Machine/Morphology/WordAnalysis.cs`), and HermitCrab's + internal `Word`. Say which. Likewise "grammar" is the HermitCrab configuration + as a whole and has no `Grammar` type - prefer `Language` or `Stratum`; "shape" + is a phonological form, not geometry; "analysis" is morphological decomposition + unless you name another domain; and a "reference" is a Scripture or row + location, never object identity. + +## Changing code + +- Add or update focused tests with every behavior change. Keep fixtures + deterministic and platform assumptions explicit. +- Use ordinal comparison for markers, tokens, identifiers, and protocol text; + culture-sensitive only where the operation is genuinely linguistic. This is the + defect class that ships here most often. +- Dispose engines, models, trainers, and streams according to their contracts, + and be explicit about who owns a stream that is passed in. +- Preserve public API semantics unless the change intends otherwise and updates + the tests. The published libraries are consumed as `netstandard2.0`. +- Prefer an existing abstraction to a parallel one. + +## Branch hygiene + +Preserve unrelated changes and pre-existing untracked files. Never use +`reset --hard`, `checkout --`, broad deletion, or broad staging as a cleanup +shortcut. Local review notes go in `.review/`, which `.gitignore` excludes. + +When a change ports work from `sillsdev/machine.py`, link the source pull +request. A workflow files the porting issue after merge; do not hand-file a +duplicate. + +## Agent guidance + +`CLAUDE.md` imports this file. Claude workflows live under `.claude/skills/`. +Path-scoped review rules live under `docs/review/`; match the changed path, +first row wins: + +| Path glob | Rules file | +| --- | --- | +| `src/SIL.Machine/Corpora/**/*.cs` | `docs/review/corpora-usfm.md` | +| `src/SIL.Machine/PunctuationAnalysis/**/*.cs` | `docs/review/punctuation.md` | +| `src/SIL.Machine.Morphology.HermitCrab/**/*.cs` | `docs/review/hermitcrab.md` | +| any other `src/**/*.cs` | `docs/review/machine-library.md` | +| `tests/**/*.cs` | `docs/review/machine-tests.md` | + +`docs/review/devils-advocate.md` is an optional adversarial pass for a high-risk +change. Add a nested `AGENTS.md` only when a subtree needs different rules. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..ed5df484d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,7 @@ +@AGENTS.md + +## Claude Code + +- Repository-wide guidance is `AGENTS.md`, imported above. +- Claude-only workflows live under `.claude/skills/`; path-scoped review rules + under `docs/review/`; `.github/` holds only workflows and templates. diff --git a/docs/review/corpora-usfm.md b/docs/review/corpora-usfm.md new file mode 100644 index 000000000..3de291eaa --- /dev/null +++ b/docs/review/corpora-usfm.md @@ -0,0 +1,22 @@ +# Corpora and USFM Review + +*Review corpus and USFM changes for deterministic marker/token behavior, ScriptureRef +and versification correctness, Unicode handling, and safe file inputs.* + +Governs `src/SIL.Machine/Corpora/**/*.cs`. + +- Trace changed behavior from source text/file or corpus row through tokenization, + parsing, ScriptureRef/`ScrVers` conversion, update handling, and emitted text. +- Use ordinal comparison for marker, token, identifier, and protocol identity unless the + code's contract is explicitly linguistic or user-facing. Do not blanket-replace + culture-aware comparisons; justify the semantic choice. +- Reference and versification arithmetic is the defect class that recurs most here + (`4e889539`, `54687760`, `8d924c1a`, `f9ba7bb7`, `78350670`). For any change that + touches it, add paired input/output or reference assertions over the affected + book/chapter/verse mapping, marker nesting, empty and malformed input, and the + relevant Unicode case. +- Check that missing, duplicate, or ambiguous references fail or resolve according to + the existing contract. Do not treat a parser snapshot as proof of visual rendering + parity. +- For files, ZIPs, and streams, preserve entry/byte limits, path validation, disposal, + cancellation, and actionable errors. diff --git a/docs/review/devils-advocate.md b/docs/review/devils-advocate.md new file mode 100644 index 000000000..047d0c5e5 --- /dev/null +++ b/docs/review/devils-advocate.md @@ -0,0 +1,25 @@ +# Machine Devil's Advocate + +*Optional adversarial second pass for a high-risk change.* + +Read the merge-base diff, the normal review output, and the tests. Change +nothing. + +This pass prunes the first review; it does not sweep for new categories. Take +the findings it produced and make each earn its place. + +The threshold: a concern is a finding only when the evidence is already in hand. +Label each item `Verified objection` or `Unverified question`, and say what +evidence would settle an unverified one instead of supplying the answer. Drop +what survives neither label. + +Challenge the most consequential claim first, one objection at a time, each with +a `path:line` and a concrete scenario. In this repository the claims that have +failed before are: + +- a HermitCrab performance win asserted without a measured artifact, or a memo + key that omits a field a rule reads; +- a USFM or reference change whose test proves the happy path only; +- a parity claim about `machine.py` with no checked comparison. + +Close with `Top risk`, `Evidence still needed`, and `Would this block merge?`. diff --git a/docs/review/hermitcrab.md b/docs/review/hermitcrab.md new file mode 100644 index 000000000..da050372b --- /dev/null +++ b/docs/review/hermitcrab.md @@ -0,0 +1,22 @@ +# HermitCrab Review + +*Review HermitCrab morphology changes for analysis equivalence, memoization-key +completeness, retained-memory bounds, parallelism, and hot-path cost.* + +Governs `src/SIL.Machine.Morphology.HermitCrab/**/*.cs`. + +- Treat analysis output as the primary contract. A faster parse, more memo hits, or a + successful build does not prove equivalent analyses. +- When changing analysis-side rules or state, re-audit every field the rule reads + against `AnalysisStateKey`. Check freezing, cached hashes, mutable dictionaries, + equality, rule counts, non-head counts, feature structures, and stratum identity. +- Memoized results must represent fully expanded subtrees. Check replay prefixes, + deduplication, empty/nogood entries, in-flight recursion, and the separation between + sequential and parallel scopes. +- Do not weaken an existing memo or retained-word bound without measured evidence + and a test. Read the current limits from the code. +- Inspect allocations and retained object lifetimes only in changed inner loops. If the + change claims a performance improvement, require a reproducible benchmark or measured + artifact in addition to semantic regression tests. +- Exercise sequential and parallel behavior where the changed path supports both, and + test cancellation/disposal if a boundary is asynchronous. diff --git a/docs/review/machine-library.md b/docs/review/machine-library.md new file mode 100644 index 000000000..8b22beaef --- /dev/null +++ b/docs/review/machine-library.md @@ -0,0 +1,25 @@ +# Machine Library Review + +*Review shipped library code for compatibility, deterministic comparison, async +contracts, and disposal.* + +Governs any `src/**/*.cs` no more specific rules file claims. + +- This is shipped library code, consumed as `netstandard2.0`. Check public and + protected API shape, overloads, optional parameters, and return types when they + change; do not introduce an API that silently drops existing consumers. +- Use ordinal comparison for markers, tokens, identifiers, and protocol text. + Reserve culture-sensitive comparison for genuinely linguistic operations. This + is the defect class review misses most often here: `075c6ea1`, `dac2d895`, and + `418ff225` all shipped it. +- Existing async APIs use `Task` and `CancellationToken`; corpus and tokenizer + APIs are synchronous. In changed async code verify cancellation propagation, + ordering, and disposal. +- Dispose engines, models, trainers, and streams according to their contracts, + and check who owns a stream that is passed in - `5a488c01` and `37e13b79` were + both ownership bugs. +- Test projects enable nullable reference types and the shipped libraries do + not. Review a changed annotation for a false promise; do not ask for a + repository-wide migration. +- Add focused tests for the decisions the diff changes. Report the commands you + ran; a coverage percentage is not evidence. diff --git a/docs/review/machine-tests.md b/docs/review/machine-tests.md new file mode 100644 index 000000000..04d41c4b9 --- /dev/null +++ b/docs/review/machine-tests.md @@ -0,0 +1,22 @@ +# Machine Tests Review + +*Review machine tests as evidence for changed behavior, edge cases, contracts, and +resource/cancellation boundaries.* + +Governs `tests/**/*.cs`. + +- A test must exercise the changed behavior, not merely execute the changed method. + Identify branches, guards, ordering, error paths, cancellation, limits, and side + effects in the production diff. +- Prefer focused NUnit tests in the affected test project. Keep fixtures deterministic + and avoid sleeps, ambient machine state, or localized selectors. +- For string/token/marker/USFM behavior, include relevant Unicode, empty, malformed, + nested, and non-English-culture cases. Do not use a culture-sensitive assertion helper + when the contract is ordinal identity. +- For async code, assert cancellation and completion behavior where the change promises + it; do not hide unobserved tasks. +- For HermitCrab changes, compare analysis semantics, not only memo-hit counts or + execution success. Exercise key completeness, replay, resource caps, and + parallel/sequential equivalence when touched. +- Name the test that proves the change. "Where is the test?" is the single most + common review question in this repository; answer it before it is asked. diff --git a/docs/review/punctuation.md b/docs/review/punctuation.md new file mode 100644 index 000000000..0395605a0 --- /dev/null +++ b/docs/review/punctuation.md @@ -0,0 +1,24 @@ +# Punctuation Analysis Review + +*Review quotation-mark and punctuation analysis for Unicode safety, malformed +input, and chapter/verse bookkeeping.* + +Governs `src/SIL.Machine/PunctuationAnalysis/**/*.cs`. + +This area's history is almost entirely crash fixes, so review it for the input +that should not have reached the code rather than for the happy path. + +- Index by text element, not by `char`. A surrogate pair, a combining mark, or a + multi-byte quotation mark must not split - `03621d14` fixed a crash from + exactly that. +- Assume the chapter or verse is missing, out of range, or unparsable. The + resolver runs over real Paratext projects: `36a24b57` fixed a crash on an + invalid chapter and `f9ba7bb7` fixed chapter numbers that came back wrong. +- A depth or state machine that tracks open and close marks must terminate on + unbalanced input and say what it saw, rather than running to the end of the + text. +- Treat quotation-mark identity as ordinal. Denormalization maps one code point + to another; a culture-aware comparison here is a bug. +- Add a fixture for the malformed case with the fix, in + `tests/SIL.Machine.Tests/PunctuationAnalysis/`. Every fix above was reported + from live data, not found by review. diff --git a/local_check.sh b/local_check.sh index 44d78c087..2b9717dad 100755 --- a/local_check.sh +++ b/local_check.sh @@ -1,4 +1,31 @@ #!/bin/bash +# Usage: ./local_check.sh [--agent-strict] +# +# --agent-strict adds the comment-hygiene check and fails on a violation in the +# lines this branch adds. Agents must pass it; humans need not. + +agent_strict=false +if [ "${1:-}" = "--agent-strict" ]; then + agent_strict=true + shift +fi + +if [ "$#" -ne 0 ]; then + echo "Usage: $0 [--agent-strict]" >&2 + exit 2 +fi + +if [ "$agent_strict" = true ]; then + if ! command -v pwsh > /dev/null 2>&1; then + echo "--agent-strict needs PowerShell 7 (pwsh) on PATH." >&2 + exit 1 + fi + pwsh -NoProfile -File ./scripts/comment-hygiene.ps1 + if [ $? -ne 0 ]; then + exit 1 + fi +fi + dotnet tool restore dotnet restore dotnet csharpier check . diff --git a/scripts/CommentHygiene.psm1 b/scripts/CommentHygiene.psm1 new file mode 100644 index 000000000..6229042a4 --- /dev/null +++ b/scripts/CommentHygiene.psm1 @@ -0,0 +1,356 @@ +<# +.SYNOPSIS + Comment-hygiene rules for the machine repository. + +.DESCRIPTION + Classifies whole-line comments and applies the banned-content categories, + the .editorconfig display-width limit, and the aggregate + implementation-comment budget. The rules stand apart from any file list or + diff scope, so they can be exercised on their own. +#> + +Set-StrictMode -Version Latest + +# Declared at module scope: under Set-StrictMode a script-scoped variable read +# before its first assignment throws rather than returning $null. +$script:editorConfigCache = @{} + +function Get-NonAsciiPunctuationCharacters { + <# + .SYNOPSIS + Returns the set of typographic characters the banned-punctuation rule reports. + + .DESCRIPTION + Built from [char] code points rather than escape sequences so the set is + identical under every PowerShell edition that can run this module. Only + these characters are reported; comments may otherwise contain any script, + IPA, or Unicode content the language data requires. + #> + return @( + [char]0x2014, [char]0x2013, [char]0x2192, [char]0x2190, [char]0x2194, + [char]0x2026, [char]0x2022, [char]0x00d7, [char]0x2018, [char]0x2019, + [char]0x201c, [char]0x201d, [char]0x00a7 + ) +} + +function Get-NonAsciiPunctuationPattern { + <# + .SYNOPSIS + Returns a regex matching any single character in Get-NonAsciiPunctuationCharacters. + #> + $escaped = Get-NonAsciiPunctuationCharacters | ForEach-Object { [regex]::Escape([string]$_) } + return '(?:' + ($escaped -join '|') + ')' +} + +function Get-CommentHygieneCategories { + <# + .SYNOPSIS + Returns the ordered category-name to regex-pattern map. + + .DESCRIPTION + Each pattern targets a specific low-signal comment shape. "Stage N" and + "this commit" are not banned: HermitCrab strata and Commit are domain + terms here. + #> + $absenceNarration = '(?i)\b(?:it|this|that|these|those|we|they|which)\s+used to\b' ` + + '|\bused to be\b|\bpreviously (?:read|worked|did|returned|used|called)\b' ` + + '|\b(?:was|were) removed\b|\b(?:was|were) stale\b|\brenamed from\b|\bfirst shipped\b' ` + + '|\bno longer (?:used|needed|exists|exist|supported|present|applies|apply|valid)\b' + $provenance = '(?i)\bshared by \w+ and \w+\b|\bthe only caller\b|\bthe sole caller\b|\bextracted from\b' + + return [ordered]@{ + 'process-framing' = '(?i)\bPhase[\s-]?\d+\b|\blater we\x27ll\b|\bwe\x27ll (?:later|eventually)\b' + 'doc-pointer' = '\b[\w./-]+\.md\b|(?i)\bsection\s+\d+[a-z]?\b' + # A bare "used to" or "no longer" usually reads as purpose or present + # state here, so both require an explicitly historical phrasing. + 'absence-narration' = $absenceNarration + 'cross-file-pointer' = "(?i)\bsee [A-Za-z]+\x27s note\b|\bas documented (?:on|in) [A-Za-z]+\b" + 'provenance' = $provenance + 'non-ascii-punctuation' = Get-NonAsciiPunctuationPattern + } +} + +function Get-CommentHygieneLanguage { + <# + .SYNOPSIS + Classifies a path into a comment-syntax family by extension. + + .OUTPUTS + 'CLike' for C# and the SentencePiece C/C++ wrapper, 'Script' for + PowerShell, shell, and Python, or $null for anything else. + #> + param([Parameter(Mandatory)][string] $Path) + + if ($Path -match '\.(ps1|psm1|sh|py)$') { return 'Script' } + if ($Path -match '\.(cs|cpp|cxx|cc|c|h|hpp)$') { return 'CLike' } + return $null +} + +function Get-CommentHygieneEditorConfig { + <# + .SYNOPSIS + Reads max_line_length and tab_width from the repository's .editorconfig [*] section. + + .DESCRIPTION + The comment width limit is whatever .editorconfig already declares for + every file, so the two cannot drift apart. Only the [*] section is read, + because that is where this repository declares the value. A language + section that later sets its own max_line_length would need section + matching here first. + + .OUTPUTS + A hashtable with MaxLineLength and TabWidth, defaulting to 120 and 4. + #> + param([Parameter(Mandatory)][string] $RepoRoot) + + if ($script:editorConfigCache.ContainsKey($RepoRoot)) { return $script:editorConfigCache[$RepoRoot] } + + $settings = @{ MaxLineLength = 120; TabWidth = 4 } + $path = Join-Path $RepoRoot '.editorconfig' + if (Test-Path -LiteralPath $path) { + $inStarSection = $false + foreach ($raw in [System.IO.File]::ReadAllLines($path, [System.Text.Encoding]::UTF8)) { + $line = $raw.Trim() + if ($line.StartsWith('#') -or $line.Length -eq 0) { continue } + if ($line.StartsWith('[')) { $inStarSection = ($line -eq '[*]'); continue } + if (-not $inStarSection) { continue } + if ($line -match '^max_line_length\s*=\s*(\d+)$') { $settings.MaxLineLength = [int]$Matches[1] } + if ($line -match '^tab_width\s*=\s*(\d+)$') { $settings.TabWidth = [int]$Matches[1] } + } + } + + $script:editorConfigCache[$RepoRoot] = $settings + return $settings +} + +function Get-CommentDisplayWidth { + <# + .SYNOPSIS + Returns a line's width in display columns, expanding tabs to the next tab stop. + + .DESCRIPTION + String length counts a tab as one character while max_line_length counts + display columns. The two disagree by enough to decide a violation either + way in tab-indented files. + #> + param( + [Parameter(Mandatory)][AllowEmptyString()][string] $Line, + [Parameter(Mandatory)][int] $TabWidth + ) + + $width = 0 + foreach ($char in $Line.ToCharArray()) { + if ($char -eq "`t") { $width += $TabWidth - ($width % $TabWidth) } + else { $width++ } + } + return $width +} + +function Get-CommentLineClassification { + <# + .SYNOPSIS + Classifies every line as an implementation comment, an exempt doc comment, or neither. + + .PARAMETER Lines + The file's lines. + + .PARAMETER Language + 'CLike' for the // and /// forms, or 'Script' for the number-sign form + and the PowerShell block-comment form. + + .OUTPUTS + A hashtable with parallel arrays Kinds ('impl', 'exempt', or $null per + line) and Bodies (comment text per line, or $null). An exempt line is + outside the aggregate budget but still subject to width and content + rules. A shebang is not a comment. A bare # is never a comment in a + CLike file, so a preprocessor directive is never misread as one. + #> + param( + [Parameter(Mandatory)][AllowEmptyCollection()][AllowEmptyString()][string[]] $Lines, + [Parameter(Mandatory)][ValidateSet('CLike', 'Script')][string] $Language + ) + + $kinds = New-Object 'object[]' $Lines.Count + $bodies = New-Object 'object[]' $Lines.Count + $inHelpBlock = $false + + for ($i = 0; $i -lt $Lines.Count; $i++) { + $trimmed = $Lines[$i].Trim() + + if ($Language -eq 'Script') { + if ($inHelpBlock) { + $kinds[$i] = 'exempt' + $bodies[$i] = $trimmed + if ($trimmed.EndsWith('#>')) { $inHelpBlock = $false } + continue + } + if ($trimmed.StartsWith('<#')) { + $kinds[$i] = 'exempt' + $bodies[$i] = $trimmed.Substring(2).TrimEnd('#', '>', ' ') + if (-not $trimmed.EndsWith('#>')) { $inHelpBlock = $true } + continue + } + if ($i -eq 0 -and $trimmed.StartsWith('#!')) { + $kinds[$i] = $null + $bodies[$i] = $null + continue + } + if ($trimmed.StartsWith('#')) { + $kinds[$i] = 'impl' + $bodies[$i] = $trimmed.Substring(1) + continue + } + } + else { + if ($trimmed.StartsWith('///')) { + $kinds[$i] = 'exempt' + $bodies[$i] = $trimmed.Substring(3) + continue + } + if ($trimmed.StartsWith('//')) { + $kinds[$i] = 'impl' + $bodies[$i] = $trimmed.Substring(2) + continue + } + } + + $kinds[$i] = $null + $bodies[$i] = $null + } + + return @{ Kinds = $kinds; Bodies = $bodies } +} + +function Get-CommentHygieneViolations { + <# + .SYNOPSIS + Scans the given files for mechanical comment-hygiene violations. + + .PARAMETER Files + Paths to scan. A file whose extension Get-CommentHygieneLanguage does not + recognize is skipped. + + .PARAMETER LineFilter + Optional map from a path to the set of 1-based line numbers to check. + Omit it to scan every line of every file. + + .PARAMETER RepoRoot + The root whose .editorconfig supplies the width limit. + + .OUTPUTS + One object per violation with File, Line, Category, and Text. The + 'comment-too-long' category covers a run of consecutive implementation + comment lines whose combined trimmed bodies exceed the budget; a block is + reported at its first line, and a block whose untouched lines alone + already exceed the budget is left to a separate cleanup. + #> + param( + [Parameter(Mandatory)][AllowEmptyCollection()][string[]] $Files, + [hashtable] $LineFilter, + [Parameter(Mandatory)][string] $RepoRoot + ) + + $editorConfig = Get-CommentHygieneEditorConfig -RepoRoot $RepoRoot + $categories = Get-CommentHygieneCategories + $violations = New-Object System.Collections.ArrayList + $maxImplCommentChars = 200 + + foreach ($file in $Files) { + if (-not (Test-Path -LiteralPath $file)) { continue } + + $language = Get-CommentHygieneLanguage -Path $file + if ($null -eq $language) { continue } + + $allowedLines = $null + if ($LineFilter -and $LineFilter.ContainsKey($file)) { $allowedLines = $LineFilter[$file] } + + # ReadAllLines rather than Get-Content: this scan runs on every pull + # request over every changed file, and StreamReader still honours a BOM. + $lines = [System.IO.File]::ReadAllLines($file, [System.Text.Encoding]::UTF8) + $classification = Get-CommentLineClassification -Lines $lines -Language $language + $kinds = $classification.Kinds + $bodies = $classification.Bodies + + for ($i = 0; $i -lt $lines.Count; $i++) { + if ($null -eq $kinds[$i]) { continue } + $lineNumber = $i + 1 + if ($allowedLines -and -not $allowedLines.Contains($lineNumber)) { continue } + + foreach ($category in $categories.Keys) { + if ($bodies[$i] -match $categories[$category]) { + [void]$violations.Add([PSCustomObject]@{ + File = $file + Line = $lineNumber + Category = $category + Text = $bodies[$i].Trim() + }) + } + } + + # Width comes from .editorconfig and applies to every comment line + # including doc comments: those are exempt from what they may say, + # not from how wide they may run. + $displayWidth = Get-CommentDisplayWidth -Line $lines[$i] -TabWidth $editorConfig.TabWidth + if ($displayWidth -gt $editorConfig.MaxLineLength) { + [void]$violations.Add([PSCustomObject]@{ + File = $file + Line = $lineNumber + Category = 'comment-line-too-long' + Text = ("{0} columns (max {1}): {2}" -f + $displayWidth, $editorConfig.MaxLineLength, $bodies[$i].Trim()) + }) + } + } + + $blockStart = -1 + $blockLength = 0 + for ($i = 0; $i -le $lines.Count; $i++) { + $isImplLine = ($i -lt $lines.Count) -and ($kinds[$i] -eq 'impl') + if ($isImplLine) { + if ($blockStart -lt 0) { $blockStart = $i } + $blockLength++ + continue + } + + if ($blockLength -gt 0) { + $blockIndexes = $blockStart..($blockStart + $blockLength - 1) + $totalChars = [int]( + $blockIndexes | ForEach-Object { $bodies[$_].Trim().Length } | Measure-Object -Sum + ).Sum + + if ($totalChars -gt $maxImplCommentChars) { + $touchesBlock = $true + $untouchedChars = 0 + if ($allowedLines) { + $touchesBlock = [bool]($blockIndexes | Where-Object { $allowedLines.Contains($_ + 1) }) + $untouchedIndexes = $blockIndexes | Where-Object { -not $allowedLines.Contains($_ + 1) } + if ($untouchedIndexes) { + $untouchedChars = [int]( + $untouchedIndexes | ForEach-Object { $bodies[$_].Trim().Length } | Measure-Object -Sum + ).Sum + } + } + if ($touchesBlock -and $untouchedChars -le $maxImplCommentChars) { + [void]$violations.Add([PSCustomObject]@{ + File = $file + Line = $blockStart + 1 + Category = 'comment-too-long' + Text = ("{0} chars (budget {1}): {2}" -f + $totalChars, $maxImplCommentChars, $bodies[$blockStart].Trim()) + }) + } + } + } + $blockStart = -1 + $blockLength = 0 + } + } + + # The unary comma stops the pipeline unrolling an empty or single-element + # array into $null or a bare scalar. + return , $violations.ToArray() +} + +Export-ModuleMember -Function @( + 'Get-CommentHygieneViolations' +) diff --git a/scripts/comment-hygiene.ps1 b/scripts/comment-hygiene.ps1 new file mode 100644 index 000000000..c33e6adb0 --- /dev/null +++ b/scripts/comment-hygiene.ps1 @@ -0,0 +1,340 @@ +#!/usr/bin/env pwsh +<# +.SYNOPSIS + Checks the comment hygiene of the lines this branch adds. + +.DESCRIPTION + Diffs the working tree against the merge base with the base ref, collects the + added lines in C#, C/C++, PowerShell, shell, and Python files, and applies the + rules in CommentHygiene.psm1. Untracked in-scope files are treated as entirely + new. + + Scoping to added lines is what makes the check usable: existing comment debt + stays out of the way, and a branch is answerable only for what it writes. + +.PARAMETER BaseRef + The ref to diff against. Defaults to the pull request base in CI, then + origin/HEAD, then origin/master. + +.PARAMETER Full + Scans every tracked in-scope file instead of the added lines. Use it to size + existing debt; it is not the pull request gate. + +.PARAMETER Advisory + Reports violations and still exits 0. CI uses this so existing debt and any + false positive cannot block a contributor. + +.PARAMETER ReportPath + Optional path for a JSON report. + +.PARAMETER SelfTest + Runs the built-in rule cases and exits. It needs no test framework, so the + rules stay verifiable on any machine that can run this script. + +.EXAMPLE + ./scripts/comment-hygiene.ps1 + Checks the lines this branch adds and fails on a violation. + +.EXAMPLE + ./scripts/comment-hygiene.ps1 -Full -Advisory + Reports all existing comment debt without failing. +#> +[CmdletBinding()] +param( + [string] $BaseRef, + [switch] $Full, + [switch] $Advisory, + [string] $ReportPath, + [switch] $SelfTest +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +Import-Module (Join-Path $PSScriptRoot 'CommentHygiene.psm1') -Force + +$scopedPathSpecs = @( + 'src/*.cs', 'tests/*.cs', + 'src/sentencepiece4c/*.cpp', 'src/sentencepiece4c/*.cxx', 'src/sentencepiece4c/*.cc', + 'src/sentencepiece4c/*.c', 'src/sentencepiece4c/*.h', 'src/sentencepiece4c/*.hpp', + 'scripts/*.ps1', 'scripts/*.psm1', 'scripts/*.sh', 'scripts/*.py', + 'local_check.sh' +) + +function Get-RepoRoot { + <# + .SYNOPSIS + Returns the top level of the working tree this script lives in. + #> + $root = & git rev-parse --show-toplevel 2>$null + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($root)) { + throw 'comment-hygiene must run inside a git working tree.' + } + return $root.Trim() +} + +function Resolve-BaseRef { + <# + .SYNOPSIS + Resolves the ref to diff against, preferring an explicit value. + + .DESCRIPTION + The candidates are tried in order and the first one git can resolve wins. + git remote show is deliberately not used: resolving a base must not need + network access on a developer machine. + #> + param([string] $Explicit) + + $candidates = @() + if (-not [string]::IsNullOrWhiteSpace($Explicit)) { $candidates += $Explicit } + if (-not [string]::IsNullOrWhiteSpace($env:GITHUB_BASE_REF)) { $candidates += "origin/$($env:GITHUB_BASE_REF)" } + $candidates += 'origin/HEAD' + $candidates += 'origin/master' + + foreach ($candidate in $candidates) { + & git rev-parse --verify --quiet "$candidate^{commit}" > $null 2>&1 + if ($LASTEXITCODE -eq 0) { return $candidate } + } + throw "No usable base ref. Tried: $($candidates -join ', '). Fetch the base branch, or pass -BaseRef." +} + +function Get-AddedLineFilter { + <# + .SYNOPSIS + Returns a map from an in-scope path to the set of line numbers this branch adds. + + .DESCRIPTION + Diffs the working tree against the merge base so local edits are checked + before they are committed. A deleted line is never scanned, and a tracked + line that still matches the merge base is not new. An untracked in-scope + file counts entirely as added. + #> + param([Parameter(Mandatory)][string] $Base, [Parameter(Mandatory)][string] $RepoRoot) + + $mergeBase = & git merge-base $Base HEAD 2>$null + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($mergeBase)) { + throw "No merge base between $Base and HEAD. Fetch enough history (fetch-depth: 0 in CI)." + } + $mergeBase = $mergeBase.Trim() + + $filter = @{} + $currentFile = $null + $hunkStart = 0 + $hunkOffset = 0 + $diff = & git diff --unified=0 $mergeBase -- @scopedPathSpecs + foreach ($line in $diff) { + if ($line.StartsWith('+++ ')) { + $path = $line.Substring(4).Trim() + if ($path -eq '/dev/null') { $currentFile = $null; continue } + if ($path.StartsWith('b/')) { $path = $path.Substring(2) } + $currentFile = (Join-Path $RepoRoot $path) + if (-not $filter.ContainsKey($currentFile)) { + $filter[$currentFile] = New-Object 'System.Collections.Generic.HashSet[int]' + } + continue + } + if ($line -match '^@@ -\S+ \+(\d+)(?:,(\d+))? @@') { + $hunkStart = [int]$Matches[1] + $hunkOffset = 0 + continue + } + if ($null -ne $currentFile -and $line.StartsWith('+')) { + [void]$filter[$currentFile].Add($hunkStart + $hunkOffset) + $hunkOffset++ + } + } + + $untracked = & git ls-files --others --exclude-standard -- @scopedPathSpecs + foreach ($path in $untracked) { + if ([string]::IsNullOrWhiteSpace($path)) { continue } + $full = Join-Path $RepoRoot $path.Trim() + if (-not (Test-Path -LiteralPath $full)) { continue } + $set = New-Object 'System.Collections.Generic.HashSet[int]' + $count = [System.IO.File]::ReadAllLines($full, [System.Text.Encoding]::UTF8).Count + for ($i = 1; $i -le $count; $i++) { [void]$set.Add($i) } + $filter[$full] = $set + } + + return @{ MergeBase = $mergeBase; Filter = $filter } +} + +function Write-Violation { + <# + .SYNOPSIS + Prints one violation, adding a GitHub annotation when running advisory in Actions. + #> + param([Parameter(Mandatory)][psobject] $Violation, [Parameter(Mandatory)][string] $RepoRoot) + + # Join-Path normalises to the platform separator while git reports forward + # slashes, so both sides are normalised before the prefix is removed. + $relative = $Violation.File -replace '\\', '/' + $root = $RepoRoot -replace '\\', '/' + if ($relative.StartsWith($root, [System.StringComparison]::OrdinalIgnoreCase)) { + $relative = $relative.Substring($root.Length).TrimStart('/') + } + Write-Host ("{0}:{1}: {2}: {3}" -f $relative, $Violation.Line, $Violation.Category, $Violation.Text) + if ($Advisory -and $env:GITHUB_ACTIONS -eq 'true') { + Write-Host ("::warning file={0},line={1}::{2}: {3}" -f + $relative, $Violation.Line, $Violation.Category, $Violation.Text) + } +} + +function Invoke-SelfTest { + <# + .SYNOPSIS + Exercises the rules against fixtures in a temporary directory. + + .OUTPUTS + 0 when every case holds, 1 otherwise. + #> + $root = Join-Path ([System.IO.Path]::GetTempPath()) ("comment-hygiene-selftest-" + [guid]::NewGuid()) + New-Item -ItemType Directory -Path $root | Out-Null + $editorConfigContent = "[*]`nmax_line_length = 120`ntab_width = 4`n" + Set-Content -LiteralPath (Join-Path $root '.editorconfig') -Value $editorConfigContent -NoNewline + $failures = New-Object System.Collections.ArrayList + + function Test-Case { + param([string] $Name, [string] $FileName, [string[]] $ContentLines, [string[]] $ExpectedCategories) + + $content = ($ContentLines -join "`n") + "`n" + $path = Join-Path $root $FileName + Set-Content -LiteralPath $path -Value $content -NoNewline + $result = Get-CommentHygieneViolations -Files @($path) -RepoRoot $root + $found = @($result | ForEach-Object { $_.Category } | Sort-Object) + $expected = @($ExpectedCategories | Sort-Object) + if (($found -join ',') -ne ($expected -join ',')) { + [void]$failures.Add( + ("{0}: expected [{1}] but found [{2}]" -f $Name, ($expected -join ','), ($found -join ',')) + ) + } + } + + $long = 'x' * 130 + $sixty = 'y' * 60 + + Test-Case 'clean C# is clean' 'clean.cs' @('// Guards against a null stratum.', 'int a = 1;') @() + Test-Case 'doc comment is budget exempt' 'doc.cs' @( + "/// $sixty", + "/// $sixty", + "/// $sixty", + "/// $sixty", + 'int a = 1;' + ) @() + Test-Case 'block budget aggregates lines' 'block.cs' @( + "// $sixty", "// $sixty", "// $sixty", "// $sixty", 'int a = 1;' + ) @('comment-too-long') + $fifty = 'z' * 50 + Test-Case 'exactly at budget is clean' 'exact.cs' @( + "// $fifty", "// $fifty", "// $fifty", "// $fifty", 'int a = 1;' + ) @() + Test-Case 'one over budget is reported' 'over.cs' @( + "// $fifty", "// $fifty", "// $fifty", "// ${fifty}a", 'int a = 1;' + ) @('comment-too-long') + Test-Case 'blank line ends a block' 'split.cs' @( + "// $sixty", "// $sixty", '', "// $sixty", "// $sixty", 'int a = 1;' + ) @() + Test-Case 'width counts the marker and indent' 'wide.cs' @("`t`t// $long", 'int a = 1;') @('comment-line-too-long') + Test-Case 'em dash is reported' 'dash.cs' @( + ("// Uses a dash " + [char]0x2014 + " here."), 'int a = 1;' + ) @('non-ascii-punctuation') + Test-Case 'absence narration is reported' 'absence.cs' @( + '// This field is no longer used by the loader.', 'int a = 1;' + ) @('absence-narration') + Test-Case 'markdown pointer is reported' 'pointer.cs' @( + '// See design-notes.md for the rationale.', 'int a = 1;' + ) @('doc-pointer') + Test-Case 'provenance is reported' 'prov.cs' @( + '// Extracted from the old tokenizer.', 'int a = 1;' + ) @('provenance') + Test-Case 'preprocessor directive is not a comment' 'pre.cs' @( + '#nullable enable', '#region Parsing', 'int a = 1;' + ) @() + Test-Case 'shell shebang is exempt' 'run.sh' @('#!/bin/bash', 'echo hi') @() + Test-Case 'powershell help block is budget exempt' 'help.ps1' @( + '<#', '.SYNOPSIS', " $sixty", " $sixty", " $sixty", " $sixty", '#>', '$a = 1' + ) @() + Test-Case 'content rules still apply inside a help block' 'helpbad.ps1' @( + '<#', '.DESCRIPTION', " Extracted from the old tokenizer.", '#>', '$a = 1' + ) @('provenance') + Test-Case 'powershell comment uses the same budget' 'budget.ps1' @( + "# $sixty", "# $sixty", "# $sixty", "# $sixty", '$a = 1' + ) @('comment-too-long') + Test-Case 'a one-line help block ends a block' 'split.ps1' @( + "# $sixty", "# $sixty", '<# Reads one line. #>', "# $sixty", "# $sixty", '$a = 1' + ) @() + Test-Case 'a module file is scanned like a script' 'rules.psm1' @( + '# This helper is no longer used by the loader.', '$a = 1' + ) @('absence-narration') + Test-Case 'shell comment uses the same budget' 'budget.sh' @( + "# $sixty", "# $sixty", "# $sixty", "# $sixty", 'echo hi' + ) @('comment-too-long') + Test-Case 'unscoped extension is skipped' 'notes.md' @('// no longer relevant') @() + + Remove-Item -LiteralPath $root -Recurse -Force + if ($failures.Count -gt 0) { + Write-Host "comment-hygiene self-test FAILED" + $failures | ForEach-Object { Write-Host " $_" } + return 1 + } + Write-Host "comment-hygiene self-test passed" + return 0 +} + +if ($SelfTest) { exit (Invoke-SelfTest) } + +$repoRoot = Get-RepoRoot +$lineFilter = $null +$scanFiles = @() +$scopeDescription = '' + +if ($Full) { + $tracked = & git ls-files -- @scopedPathSpecs + $scanFiles = @($tracked | Where-Object { $_ } | ForEach-Object { Join-Path $repoRoot $_.Trim() }) + $scopeDescription = "all $($scanFiles.Count) tracked in-scope files" +} +else { + $base = Resolve-BaseRef -Explicit $BaseRef + $added = Get-AddedLineFilter -Base $base -RepoRoot $repoRoot + $lineFilter = $added.Filter + $scanFiles = @($lineFilter.Keys) + $scopeDescription = "lines added since $base (merge base $($added.MergeBase.Substring(0, 8)))" +} + +if ($scanFiles.Count -eq 0) { + Write-Host "comment-hygiene: no in-scope files to check ($scopeDescription)." + exit 0 +} + +$violations = Get-CommentHygieneViolations -Files $scanFiles -LineFilter $lineFilter -RepoRoot $repoRoot + +if (-not [string]::IsNullOrWhiteSpace($ReportPath)) { + $reportDir = Split-Path -Parent $ReportPath + if ($reportDir -and -not (Test-Path -LiteralPath $reportDir)) { + New-Item -ItemType Directory -Path $reportDir -Force | Out-Null + } + $payload = [PSCustomObject]@{ + scope = $scopeDescription + advisory = [bool]$Advisory + violationCount = $violations.Count + violations = $violations + } + $payload | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $ReportPath -Encoding utf8 +} + +if ($violations.Count -eq 0) { + Write-Host "comment-hygiene: clean ($scopeDescription)." + exit 0 +} + +Write-Host "comment-hygiene: $($violations.Count) violation(s) in $scopeDescription." +Write-Host '' +foreach ($violation in $violations) { Write-Violation -Violation $violation -RepoRoot $repoRoot } +Write-Host '' +Write-Host 'The standard is .claude/skills/code-comments/SKILL.md.' +Write-Host 'Reproduce locally with: pwsh ./scripts/comment-hygiene.ps1' + +if ($Advisory) { + Write-Host 'Advisory only; not failing the run.' + exit 0 +} +exit 1