diff --git a/.quality/evidence/002-project-graph/quality-map.yaml b/.quality/evidence/002-project-graph/quality-map.yaml index a2689ff..537320d 100644 --- a/.quality/evidence/002-project-graph/quality-map.yaml +++ b/.quality/evidence/002-project-graph/quality-map.yaml @@ -10,7 +10,7 @@ target: aliases: [] source_refs: - path: "AGENTS.md" - label: "Agents must not set human ratification fields or accept risk for a human" + label: "Agents must not set human validation fields or accept risk for a human" - path: "agent-skills/quality/references/_shared/layout.md" label: "Graph artifact layout and per-command edit ownership" @@ -171,7 +171,7 @@ expectations: source_type: "SOURCE" source_refs: - path: "AGENTS.md" - anchor: "Agents must not set human ratification fields or accept risk for a human" + anchor: "Agents must not set human validation fields or accept risk for a human" category: "compliance" priority: "UNKNOWN" evidence: diff --git a/.quality/evidence/003-runtime-observations/quality-map.yaml b/.quality/evidence/003-runtime-observations/quality-map.yaml index 7c889dc..0885aaf 100644 --- a/.quality/evidence/003-runtime-observations/quality-map.yaml +++ b/.quality/evidence/003-runtime-observations/quality-map.yaml @@ -13,7 +13,7 @@ target: - path: "AGENTS.md" label: "Independence: producers write facts, Quality reads and evaluates them" - path: "docs/concepts/trust-boundaries.md" - label: "Proof remains independent and scores are deterministic" + label: "Evidence remains independent and scores are deterministic" - path: "packages/core/src/observations/quality-observations.schema.json" label: "Canonical observation manifest schema" @@ -371,7 +371,7 @@ expectations: - path: "AGENTS.md" anchor: "Independence" - path: "docs/concepts/trust-boundaries.md" - anchor: "Proof remains independent" + anchor: "Evidence remains independent" category: "security" priority: "UNKNOWN" evidence: @@ -540,7 +540,7 @@ expectations: source_type: "SOURCE" source_refs: - path: "AGENTS.md" - anchor: "Agents must not set human ratification fields or accept risk for a human" + anchor: "Agents must not set human validation fields or accept risk for a human" category: "compliance" priority: "UNKNOWN" evidence: diff --git a/.quality/evidence/004-quality-scoring/quality-map.yaml b/.quality/evidence/004-quality-scoring/quality-map.yaml index 8876b35..9ff25ac 100644 --- a/.quality/evidence/004-quality-scoring/quality-map.yaml +++ b/.quality/evidence/004-quality-scoring/quality-map.yaml @@ -170,7 +170,7 @@ expectations: source_type: "SOURCE" source_refs: - path: "AGENTS.md" - anchor: "Agents must not set human ratification fields or accept risk for a human" + anchor: "Agents must not set human validation fields or accept risk for a human" - path: "docs/concepts/trust-boundaries.md" anchor: "Human decisions stay human" category: "compliance" diff --git a/.quality/evidence/006-quality-presentation/quality-map.yaml b/.quality/evidence/006-quality-presentation/quality-map.yaml index 12af206..19b4ce2 100644 --- a/.quality/evidence/006-quality-presentation/quality-map.yaml +++ b/.quality/evidence/006-quality-presentation/quality-map.yaml @@ -80,7 +80,7 @@ expectations: source_type: "SOURCE" source_refs: - path: "AGENTS.md" - anchor: "Agents must not set human ratification fields or accept risk for a human" + anchor: "Agents must not set human validation fields or accept risk for a human" - path: "README.md" anchor: "It will pause when a human needs to confirm feature boundaries, priorities, or quality checks" category: "compliance" diff --git a/AGENTS.md b/AGENTS.md index 8770a86..f7f5111 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ Quality evaluates evidence independently of the systems that produce it. - Quality reads and evaluates evidence; it does not create tests or modify producer artifacts. - Scores are computed by the deterministic engine, never by an agent or UI. -- Agents must not set human ratification fields or accept risk for a human. +- Agents must not set human validation fields or accept risk for a human. ## Dependency direction diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2263280..2088304 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -67,7 +67,7 @@ repository enforces them: injected by `apps/explorer`, which exposes only the operations the application needs. Authoring operations remain explicit engine capabilities and are not available to the Explorer interface. -- Human ratification fields are contract inputs. Agent workflows may propose +- Human validation fields are contract inputs. Agent workflows may propose changes but must not promote those fields. - Local and remote filesystem access stays behind data-source adapters. The adapters discover and normalize inputs; scoring operates on the normalized diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 923fdb7..62be847 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,8 +71,8 @@ for the right reason, so you know it covers what you think it covers. **Test the silence, not just the success.** The worst failures in this tool are quiet ones — a result that fails to connect, a value that is dropped rather than -rejected. A test proving something works often will not catch those. A test -proving the tool *complains* when something is wrong usually will. +rejected. A test showing that something works often will not catch those. A test +showing that the tool *complains* when something is wrong usually will. ## Releasing diff --git a/README.md b/README.md index 21409fa..aae8107 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,9 @@ Then open Claude Code in that repository and give it this prompt: ``` The agent will inventory the repository, propose a small quality graph for the -highest-priority feature, connect the proof that already exists, and run an -initial assessment when runtime results are available. It will pause when a -human needs to confirm feature boundaries, priorities, or quality checks. +highest-priority feature, connect the verification methods that already exist, +and run an initial assessment when runtime results are available. It will pause +when a human needs to confirm feature boundaries, priorities, or quality checks. After the first pass, inspect the proposed files under `.quality/` and answer the agent's review questions. Then continue with the next command it recommends, @@ -38,7 +38,7 @@ Want to control the first slice? Include it in the prompt: ``` The skill evaluates existing specifications, tests, workflows, reports, and -other evidence; it does not create tests or approve its own proposals. See the +other evidence; it does not create tests or validate its own proposals. See the [`quality` skill](agent-skills/quality/SKILL.md) for all commands and workflow details. @@ -49,35 +49,39 @@ observation set fetched from a CI workflow run.](docs/assets/quality-explorer-ov ## What Quality does -Quality connects product intent to independent, repeatable proof: +Quality connects product intent to independently produced evidence: ```text requirements (PRDs) - ↓ + ↓ intent validation features (specs) ↓ quality checks - ↓ -proof definitions (tests and other evidence) - ↓ -runtime observations (did the proof pass?) + ↓ verification + ├── reasoning-based verification → analysis results ─┐ + └── empirical verification → empirical observations ┴→ evidence ``` -Requirements define desired outcomes, checks state what must hold, and tests, -workflows, telemetry, or manual records provide proof. Quality maps these -relationships in `.quality/` and keeps missing or weak links visible. Agents can -propose maps; people review structure and accepted risk. +Requirements define desired outcomes, and people validate that features and +checks represent that intent. Verification methods then evaluate whether the +implementation or observed behavior satisfies each check. Reasoning-based +methods produce analysis results; empirical methods produce observations of +behavior. Both can contribute evidence. Quality maps these relationships in +`.quality/` and keeps missing or weak links visible. Agents can propose maps; +people validate intent and decide whether to accept risk. The engine reports four separate measures: | Measure | Question it answers | | --- | --- | -| Quality | Is the current proving evidence passing? | -| Coverage | Does every declared check have proof? | -| Evidence confidence | Is the mapped proof strong enough? | +| Quality | What do the current observed results report? | +| Coverage | Does every declared check have a mapped verification method? | +| Evidence confidence | Are the mapped methods strong enough for the claim? | | Structure confidence | Are these the right features, checks, and priorities? | -See [the concepts guide](docs/README.md) for the model and trust boundaries. +See the [terminology guide](docs/concepts/terminology.md) for the distinction +between validation, verification, evidence, and observation, and the +[concepts guide](docs/README.md) for the full model and trust boundaries. ## Optional: Use `spec-project` for spec-driven testing diff --git a/agent-skills/quality/SKILL.md b/agent-skills/quality/SKILL.md index 1677731..8ff7177 100644 --- a/agent-skills/quality/SKILL.md +++ b/agent-skills/quality/SKILL.md @@ -1,14 +1,14 @@ --- name: quality -description: "Construct, assess, and improve a repository's quality project graph and four-score quality index. Use only when the user explicitly invokes /quality or asks for a quality status, score, index, map, graph, or posture—not for a generic request to improve code quality. Guides new-repository setup, project and feature mapping, saved assessment scopes, proof mapping, runtime assessment, and score-directed improvement while preserving independent scoring and human ratification. Commands: start, status, map-project, map-feature, assess, improve, help." +description: "Construct, assess, and improve a repository's quality project graph and four-score quality index. Use only when the user explicitly invokes /quality or asks for a quality status, score, index, map, graph, or posture—not for a generic request to improve code quality. Guides new-repository setup, project and feature mapping, saved assessment scopes, verification-method mapping, runtime assessment, and score-directed improvement while preserving independent scoring and human validation. Commands: start, status, map-project, map-feature, assess, improve, help." --- # Quality Quality answers: -> What must this project deliver, what proves each promise, and what does the current -> proof justify believing? +> What must this project deliver, which methods verify each promise, and what does +> the current evidence justify believing? It maintains a **quality project graph** and asks the deterministic `quality-tools` engine to compute a four-score **quality index** from that graph. @@ -17,7 +17,7 @@ It never invents a score or treats a score as the goal. **Before any command, read [independence](references/_shared/independence.md).** A system cannot establish trust by grading its own claims. This principle governs every workflow, -especially evidence classification and human ratification. +especially evidence classification and human validation. ## Use one formal vocabulary @@ -33,9 +33,19 @@ Use these terms consistently: | Assessment | Project + whole-project or view scope + optional observation set + observed revision/run, producing one quality index; without an observation set the index carries the three static scores and no Quality score | | Release candidate | A concrete build, version, commit, or artifact covered by an assessment; not a persistent graph grouping | | Product | Optional descriptive language for what the project delivers; it has no graph or scoring semantics | +| Intent validation | Human confirmation that features, priorities, and checks represent accepted product intent | +| Verification method | A repeatable or inspectable way to evaluate whether implementation or observed behavior satisfies a check | +| Reasoning-based verification | Evaluation by reasoning about an implementation or model, including review, static analysis, model checking, and formal verification | +| Empirical verification | Evaluation by executing or monitoring a system, recording an observation, and comparing it with a check | +| Evidence | Information from reasoning or evaluated observations that supports or challenges a check | +| Empirical observation | A recorded fact about executed or production behavior | +| Runtime observation | Quality's standardized result record; it can carry an executable reasoning tool's outcome without making the underlying method empirical | +| Proof | A deductive argument under stated assumptions; reserve this term for formal or deductive proof | Do not introduce a second grouping such as `release area`. Use a view when a reusable subset of features must be assessed together. +Use **evidence gap** in prose; `proof_gap` is only the legacy literal field name +in the current quality-map contract. ## The quality project graph @@ -46,24 +56,24 @@ repository (storage) └── quality project └── features └── quality checks: what must hold - └── proof definitions: what could prove each check - └── runtime observations: did that proof pass? + └── verification methods: how each check is evaluated + └── runtime observations: what happened when executable methods ran? view ─────────────── selects project features observation set ──── selects runtime-result sources ``` -The graph is represented by checked-in files and the proof artifacts they +The graph is represented by checked-in files and the evidence artifacts they reference: | Graph layer | Artifact | Meaning | | --- | --- | --- | | Project → features | `.quality/project-map.yaml` | Project identity, feature boundaries, priorities, status, and dependencies | -| Feature → checks → proof | `.quality/evidence//quality-map.yaml` | What must hold for one feature and which tests, workflows, telemetry, static checks, or manual records prove it | -| Proof → runtime sources | `.quality/config/observation-sources.yaml` | Where canonical observation files come from | +| Feature → checks → verification methods | `.quality/evidence//quality-map.yaml` | What must hold for one feature and which tests, workflows, telemetry, static checks, or manual records evaluate it | +| Executable methods → runtime sources | `.quality/config/observation-sources.yaml` | Where canonical observation files come from | | Runtime source bundles | `.quality/config/observation-sets.yaml` | Which runtime sources are assessed together | | Saved assessment scopes | `.quality/config/views.yaml` | Which project-map features are included together | -| Actual proof | Files referenced by `evidence.path` | Tests, workflows, reports, runbooks, telemetry, or other auditable proof | +| Evidence artifacts | Files referenced by `evidence.path` | Tests, workflows, reports, runbooks, telemetry, or other auditable evidence | IDs, paths, and optional test-case names are the graph's edges. Generated recommendations and `fix-prompts.md` are engine output, not graph source. @@ -101,15 +111,16 @@ artifact names: brownfield, or already partially mapped. 2. **Map the project.** Construct project → feature structure and choose the highest-priority feature. -3. **Map each feature.** Define its quality checks and connect existing proof. -4. **Connect runtime.** Arrange for proof producers to publish canonical +3. **Map each feature.** Define its quality checks and connect existing + verification methods. +4. **Connect runtime.** Arrange for evidence producers to publish canonical `quality-observations.json`, within the producer edit boundary below, then configure transport-only sources and sets that locate those files. 5. **Assess.** Run `quality-tools` and explain the scores together. Coverage, evidence confidence, and structure confidence are available from step 3 onward; Quality also needs step 4. 6. **Improve.** Diagnose the weak score, improve the underlying structure, - proof, implementation, or runtime wiring, then assess again. + evidence, implementation, or runtime wiring, then assess again. 7. **Repeat.** Expand feature by feature, highest priority and risk first. `/quality start` orchestrates steps 1–5 for a new or unmapped repository. The @@ -171,10 +182,10 @@ repository's priorities. A saved assessment scope is a separate named view in | Command | Outcome | Reference | | --- | --- | --- | | `start [scope]` | Bootstrap the graph in a new, brownfield, or partially mapped repository | [start](references/start.md) | -| `status [scope]` | Read existing artifacts and recent results; do not run proof/scoring commands or edit files | [status](references/status.md) | +| `status [scope]` | Read existing artifacts and recent results; do not run verification/scoring commands or edit files | [status](references/status.md) | | `map-project [scope]` | Construct or reconcile project → feature structure | [map-project](references/map-project/index.md) | -| `map-feature ` | Construct or improve one feature's checks and proof mappings | [map-feature](references/map-feature/index.md) | -| `assess [scope]` | Refresh and explain the four scores without changing graph source or proof | [assess](references/assess.md) | +| `map-feature ` | Construct or improve one feature's checks and verification-method mappings | [map-feature](references/map-feature/index.md) | +| `assess [scope]` | Refresh and explain the four scores without changing graph source or evidence artifacts | [assess](references/assess.md) | | `improve [scope]` | Diagnose weak scores, improve the underlying system, and reassess | [improve](references/improve/index.md) | | `help [command]` | Explain commands without executing them | [help](references/help.md) | @@ -199,7 +210,7 @@ only when `start` or `map-project` requires it. - use `improve` when the user asks to raise quality posture or close gaps. 5. State what will be read, generated, or edited before acting. 6. Stop at human gates. An agent may propose structure, priorities, reviewed - checks, or accepted risk, but must never ratify for the owner. + checks, or accepted risk, but must never validate intent for the owner. ## The four scores and the honest improvement lever @@ -208,10 +219,10 @@ verdict. | Score | Question | Improve the underlying condition by | | --- | --- | --- | -| Coverage | Does every declared check have proof? | Create or map missing proof | -| Evidence confidence | Is the mapped proof strong enough? | Use a stronger appropriate modality or add a meaningful gate | -| Quality | Is current proving evidence passing? | Fix the implementation/proof, stale results, or runtime wiring and rerun it | -| Structure confidence | Are these the right features, checks, and priorities? | Ask a human to correct or ratify proposed structure | +| Coverage | Does every declared check have a mapped verification method? | Create or map a missing method | +| Evidence confidence | Are the mapped methods strong enough? | Use a stronger appropriate modality or add a meaningful gate | +| Quality | Are current observed results passing? | Fix the implementation or method, refresh stale results, or repair runtime wiring and rerun it | +| Structure confidence | Are these the right features, checks, and priorities? | Ask a human to correct or validate proposed structure | Quality is the only one of the four that needs runtime observations. When no observation set is configured or its results cannot be acquired, report the other @@ -227,7 +238,7 @@ rise. output. When no structured total exists, enumerate first and calculate the count mechanically; verify that headings, totals, and lists agree. - Group every cause under the score it actually affects. Missing or - policy-insufficient proof affects coverage; proof type and gate strength + policy-insufficient verification methods affect coverage; method type and gate strength affect evidence confidence; runtime outcomes/acquisition/resolution affect Quality; feature/check/priority provenance and review affect structure confidence. Do not cite a structure gate as an evidence-confidence cause. @@ -240,11 +251,11 @@ rise. or an authoritative artifact already contains it. Never infer attribution from Git config, environment metadata, or the operating-system account. -## Relationship to proof producers +## Relationship to evidence producers -Quality maps and judges existing proof. It may hand a concrete proof gap to a +Quality maps and judges existing evidence. It may hand a concrete evidence gap to a producer such as `/shiplight cover`, `/shiplight create-yaml-tests`, or `/shiplight create-agent-verification`. Producers create or run tests and other -proof; Quality confirms the resulting artifact, connects it to the graph, and +verification methods; Quality confirms the resulting artifact, connects it to the graph, and remeasures. The only producer-side exception is the explicitly authorized mechanical workflow glue defined above. diff --git a/agent-skills/quality/references/_shared/independence.md b/agent-skills/quality/references/_shared/independence.md index 846cd5e..5f2fc9b 100644 --- a/agent-skills/quality/references/_shared/independence.md +++ b/agent-skills/quality/references/_shared/independence.md @@ -9,7 +9,7 @@ software do what it should, and is it correct, with justified confidence? Quality answers that only if its judgment is independent of whatever produced the code and the tests. That independence is engineered, not assumed, and it is the -reason Quality is a **separate skill at a higher altitude** than the proof +reason Quality is a **separate skill at a higher altitude** than the evidence producers—never one of their commands. ## The four enforcement mechanisms (never weaken these) @@ -24,32 +24,32 @@ producers—never one of their commands. `.quality/`. 2. **The engine scores, never the LLM.** The four scores are computed by the deterministic `quality-tools` engine from declared facts + runtime observations - + human ratification — never an agent's opinion. Never blend, optimize, or + + human validation — never an agent's opinion. Never blend, optimize, or reverse-engineer a score. A measure of quality must never become the target it describes, or it stops describing anything (Goodhart). 3. **The judge re-confirms, never copies the claim.** When a producer records an evidence `type`, Quality confirms it against the artifact at `evidence.path` rather than trusting the report's self-claim — the judge does not copy the generator's label. -4. **Human ratification gate.** Structure confidence is the only human-gated - axis, ratified through four gates (see below). An agent may **construct at the +4. **Human validation gate.** Structure confidence is the only human-gated + axis, validated through four gates (see below). An agent may **construct at the untrusted default and propose** checks/priorities/structure, but must **never - self-advance a gate** or ratify on the owner's behalf. Self-verification of + self-advance a gate** or validate intent on the owner's behalf. Self-validation of structure is disallowed by design. -## Structure confidence: the ratification gates +## Structure confidence: the validation gates -Mechanism 4 is enforced through **four ratification gates** — each a separate +Mechanism 4 is enforced through **four validation gates** — each a separate field the deterministic `quality-tools` engine reads, never an agent's edit. The engine composes them into the structure-confidence score; the numbers below are the current rubric, not a target to optimize toward or reverse-engineer. -| Gate | Field | Artifact / owner | What it ratifies | +| Gate | Field | Artifact / owner | What it validates | | --- | --- | --- | --- | | 1 | `structure_provenance` | `quality-map.yaml` / `map-feature` | the check list *and its priorities* **originated** from a trusted source — `spec`/`user_authored` = 1.0, `agent_generated` = 0.7, `inferred_brownfield` = 0.4, `unspecified` = 0 (counted, earns no trust). This is *origin*, not review — review is gate 4 and never overwrites it | -| 2 | feature `status` | `project-map.yaml` / `map-project` | the feature is accepted project structure—a `candidate` (agent-proposed, unratified) feature soft-caps its checks' structure confidence at 0.7 until a human ratifies it by assigning the accurate lifecycle status, such as `planned`, `specified`, or `implemented` | +| 2 | feature `status` | `project-map.yaml` / `map-project` | the feature is accepted project structure—a `candidate` (agent-proposed, unvalidated) feature soft-caps its checks' structure confidence at 0.7 until a human validates it by assigning the accurate lifecycle status, such as `planned`, `specified`, or `implemented` | | 3 | `priority_provenance` | `project-map.yaml` / `map-project` | the declared priority is human-set (`human`) rather than agent-guessed (`agent`); the agent must not overwrite a human-set priority on rebuild | -| 4 | `checks_reviewed` | `quality-map.yaml` / `map-feature` | a human reviewed and **approved the whole check list**. Combined with a confirmed feature (gate 2), it lifts that feature's checks to HIGH structure confidence (1.0), overriding the gate-1 provenance ladder. Orthogonal to `structure_provenance` (origin) — `agent_generated` means "an agent produced the checks," *not* "a human reviewed them"; that is what this gate records | +| 4 | `checks_reviewed` | `quality-map.yaml` / `map-feature` | a human reviewed and **validated the whole check list against accepted intent**. Combined with a confirmed feature (gate 2), it lifts that feature's checks to HIGH structure confidence (1.0), overriding the gate-1 provenance ladder. Orthogonal to `structure_provenance` (origin) — `agent_generated` means "an agent produced the checks," *not* "a human reviewed them"; that is what this gate records | `map-feature` owns gates 1 and 4; `map-project` owns gates 2–3. When either command says it "owns structure confidence," it means *its* gates—the score is @@ -67,7 +67,7 @@ metadata, or the operating-system account for one. ## The two human decisions that are *not* gates Two other fields record a human decision and move a score. Neither touches -structure confidence, so **neither is a ratification gate and neither carries a +structure confidence, so **neither is a validation gate and neither carries a gate number** — the four above are the whole list. They are mirror images: | Field | The human… | Effect | @@ -77,7 +77,7 @@ gate number** — the four above are the whole list. They are mirror images: Both feed the gap reasons in `quality-structure/assessment.ts` and never the structure-confidence axis. Reporting them beside each other is the point: one -says "this needs more proof than the default", the other says "we know, and we +says "this needs stronger verification than the default", the other says "we know, and we accept it". The direction decides what an agent may do. An agent may **propose** either, and @@ -89,7 +89,7 @@ decision exactly as self-advancing a gate would. ## Layering rule -Quality may know about and drive the producers (top knows bottom — it can hand a -proof gap to `/shiplight cover` or a producer). The producers must **not** know +Quality may know about and drive the producers (top knows bottom — it can hand an +evidence gap to `/shiplight cover` or a producer). The producers must **not** know about Quality. Keep the dependency one-directional and the file interface the only coupling. diff --git a/agent-skills/quality/references/_shared/layout.md b/agent-skills/quality/references/_shared/layout.md index 52f7084..bf0c6a8 100644 --- a/agent-skills/quality/references/_shared/layout.md +++ b/agent-skills/quality/references/_shared/layout.md @@ -1,13 +1,13 @@ # Shared: The `.quality/` layout & ownership All quality-graph **source** artifacts live under `.quality/` at the repo root. -The tests, reports, workflows, telemetry, and other proof they reference remain +The tests, reports, workflows, telemetry, and other evidence they reference remain in their owning locations. Per-artifact ownership and the edit contract: ```text .quality/ ├── project-map.yaml owned by `map-project` (project and features) -├── evidence//quality-map.yaml owned by `map-feature` (per-feature checks and proof) +├── evidence//quality-map.yaml owned by `map-feature` (per-feature checks and verification methods) ├── config/ │ ├── sources.yaml AUTHOR INPUT — owned by the author and their coding agent; `map-project` reads it │ ├── observation-sources.yaml owned by `improve` @@ -39,7 +39,7 @@ Rules: No Quality command authors this file on the author's behalf. - `fix-prompts.md` and `generated/*` are written by `quality-tools` only — treat them as read-only. -- Quality reads, but does not author, the **producer proof** it indexes: +- Quality reads, but does not author, the **producer evidence** it indexes: `specs//test-spec.md`, `specs//test-report.md`, `TESTING.md`, and the test files themselves. The narrow exception is explicitly authorized workflow glue that serializes an already-determined result into canonical diff --git a/agent-skills/quality/references/_shared/vocabularies.md b/agent-skills/quality/references/_shared/vocabularies.md index 8793ed5..38cad61 100644 --- a/agent-skills/quality/references/_shared/vocabularies.md +++ b/agent-skills/quality/references/_shared/vocabularies.md @@ -5,7 +5,7 @@ reading runtime analysis. The runtime-analysis vocabularies are owned by the too never authored into hand-written artifacts. `quality-map.yaml` evidence rows carry **no authored strength vocabulary**. A -proof definition is a `type` at a `path` and optional `test_case`; **evidence +verification method is a `type` at a `path` and optional `test_case`; **evidence confidence is derived downstream from `type`**, and flakiness/staleness/pass come from runtime observations. There is no authored `depth` or `reliability`, and no hand-authored `HIGH/MEDIUM/LOW` overall-confidence verdict — the verdict diff --git a/agent-skills/quality/references/assess.md b/agent-skills/quality/references/assess.md index f0173e2..ed2a490 100644 --- a/agent-skills/quality/references/assess.md +++ b/agent-skills/quality/references/assess.md @@ -2,7 +2,7 @@ `assess` runs the deterministic engine against the existing graph. It may write engine-generated recommendation output, but it does not edit graph source, -proof artifacts, tests, workflows, or reports. +evidence artifacts, tests, workflows, or reports. ## Read first @@ -74,8 +74,9 @@ Then separate acquisition from graph resolution. The `runtime_review` block is absent when no observation set was selected: - `runtime_review.execution_status` and `profiles[]`: whether result sources - were acquired and validated -- `runtime_review.resolution_status`: whether observations joined to mapped proof + were acquired and their canonical files passed schema validation +- `runtime_review.resolution_status`: whether observations joined to mapped + verification methods - `execution_diagnostics`: source, credential, artifact, or canonical-format problems - `resolution_diagnostics` and `resolution_audit`: unmatched or ambiguous graph @@ -85,12 +86,12 @@ absent when no observation set was selected: Report the four scores side by side: -- **Coverage:** checks with mapped proof -- **Evidence confidence:** strength of mapped proof -- **Quality:** current runtime result of that proof +- **Coverage:** checks with mapped verification methods +- **Evidence confidence:** strength of mapped methods +- **Quality:** current observed results for executable methods - **Structure confidence:** trust in features, checks, and priorities -Never blend them. A high runtime score on an unratified check list is not a +Never blend them. A high runtime score on an unvalidated check list is not a high-confidence project judgment. When Quality is unavailable, report the other three as measured and state the reason from `quality_score_availability` — never mark all four unavailable because runtime data is missing. diff --git a/agent-skills/quality/references/improve/index.md b/agent-skills/quality/references/improve/index.md index d9771f5..59326a8 100644 --- a/agent-skills/quality/references/improve/index.md +++ b/agent-skills/quality/references/improve/index.md @@ -2,7 +2,7 @@ `improve` starts from an engine-produced assessment, diagnoses which underlying condition is weak, changes that condition honestly, and runs `assess` again. -The objective is better project behavior and proof—not a larger number. +The objective is better project behavior and evidence—not a larger number. ## Contents @@ -29,7 +29,7 @@ The quality graph joins structural declarations to current observations: ↓ feature id / quality_map_path .quality/evidence//quality-map.yaml ↓ evidence.path + optional evidence.test_case -proof producer / workflow +evidence producer / workflow ↓ quality-observations.json (one canonical format) .quality/config/observation-sources.yaml ↓ transport locates the canonical file @@ -44,10 +44,10 @@ The engine reports four separate scores: | Score | Diagnose | Honest improvement | | --- | --- | --- | -| Coverage | A declared check has no mapped proof | Create appropriate proof through a producer, then map it | -| Evidence confidence | Proof exists but its modality or gate is too weak for the check | Strengthen proof at the system boundary that matters | -| Quality | Proof is failing, stale, unavailable, or unobserved | Fix the implementation/proof or repair acquisition and graph joins, then rerun | -| Structure confidence | Features, checks, or priorities are inferred or unreviewed | Ask a human to correct or ratify them through `map-project` or `map-feature` | +| Coverage | A declared check has no mapped verification method | Create an appropriate method through a producer, then map it | +| Evidence confidence | A method exists but its modality or gate is too weak for the check | Strengthen verification at the system boundary that matters | +| Quality | Evidence is failing, stale, unavailable, or unobserved | Fix the implementation or method, or repair acquisition and graph joins, then rerun | +| Structure confidence | Features, checks, or priorities are inferred or unreviewed | Ask a human to correct or validate them through `map-project` or `map-feature` | No command may directly author a score. Never improve a number by removing scope, weakening checks, relabeling evidence, accepting risk, or changing @@ -82,7 +82,7 @@ Follow `assess` and retain: - recommendations in priority order Do not change anything until the low score or recommendation has been traced to -a concrete graph node, graph edge, proof artifact, or implemented behavior. +a concrete graph node, graph edge, evidence artifact, or implemented behavior. ### 2. Classify the root cause @@ -90,8 +90,8 @@ Classify each gap before editing: 1. **Structure:** wrong/missing feature, check, priority, provenance, or human review. -2. **Coverage:** no proof is mapped for a valid check. -3. **Evidence strength:** mapped proof cannot establish the full claim or lacks +2. **Coverage:** no verification method is mapped for a valid check. +3. **Evidence strength:** a mapped method cannot establish the full claim or lacks the required execution context/gate. 4. **Source acquisition:** credentials, repository/workflow selection, artifact names, or local-folder path prevent results from loading. @@ -101,7 +101,7 @@ Classify each gap before editing: status, timestamp, revision, or duplicate observation identity. 7. **Graph join:** results load but `path`/`test_case` do not match `evidence.path`/`evidence.test_case`, or the match is ambiguous. -8. **Real failure:** current proof joins correctly and reports a failing/error +8. **Real failure:** current evidence joins correctly and reports a failing/error state. 9. **Scope:** the observation set contains the wrong profiles or the saved view contains the wrong features. @@ -116,23 +116,23 @@ acquisition and resolution have been ruled out. #### Structure confidence - Follow `map-project` for feature boundaries, status, and priority provenance. -- Follow `map-feature` for the check list, proof definitions, origin provenance, +- Follow `map-feature` for the check list, verification methods, origin provenance, and whole-list review. - Propose corrections highest priority first. - Never flip a human gate or write `accepted_gaps` for the owner. #### Coverage -- Confirm the check is valid before buying proof for it. -- Use existing proof when it genuinely establishes the check and map it through +- Confirm the check is valid before commissioning a verification method for it. +- Use existing evidence when it genuinely supports the check and map its method through `map-feature`. -- Otherwise hand a precise proof gap to an appropriate producer. Quality does +- Otherwise hand a precise evidence gap to an appropriate producer. Quality does not author the test. - Confirm the resulting artifact and type before adding it to the graph. #### Evidence confidence -- Match proof strength to the claim. Unit proof may fully establish a small, +- Match method strength to the claim. Unit evidence may fully establish a small, deterministic code contract; it does not establish a cross-boundary user workflow by itself. - Add a meaningful integration, browser, smoke, telemetry, or release-gate @@ -149,18 +149,18 @@ acquisition and resolution have been ruled out. observations`; do not add a parser choice to source configuration. - For join problems, align emitted `path`/`test_case` with `evidence.path`/`evidence.test_case`. There is no second mapping table. -- For a real failure, fix the implementation or proof through its owning workflow and +- For a real failure, fix the implementation or verification method through its owning workflow and rerun it. -- For stale/unavailable proof, refresh it or report the external blocker. +- For stale/unavailable evidence, refresh it or report the external blocker. #### Scope - Fix observation-set profile membership or saved-view feature membership. - Never remove a valid high-risk feature merely to improve the result. -### 4. Validate the changed layer +### 4. Check the changed layer -- Feature map: +- Schema-validate a feature map: ```bash npx --yes @shiplightai/quality-tools@^0.3.0 validate @@ -172,7 +172,7 @@ acquisition and resolution have been ruled out. npx --yes @shiplightai/quality-tools@^0.3.0 schema ``` -- Canonical observations: +- Schema-validate canonical observations: ```bash npx --yes @shiplightai/quality-tools@^0.3.0 observations validate \ @@ -182,7 +182,7 @@ acquisition and resolution have been ruled out. - Observation config: compare with the configuration schemas in `assets/`, then run the relevant assessment. Engine diagnostics verify acquisition and graph joins. -- Implementation/proof changes: run their owning verification command before +- Implementation or verification-method changes: run their owning verification command before reassessment. Never report a fix as verified without command output or an auditable @@ -196,7 +196,7 @@ Run the same observation set and scope through `assess`. Compare like with like: - each available score before and after; a score that was unavailable in the baseline stays reported as unavailable, never as zero - recommendations closed, changed, or still open -- new evidence and the command that proved it +- new evidence and the command that produced it Repeat until remaining work is low-return, deferred, blocked by external state, or requires a human decision. @@ -252,13 +252,13 @@ the observed revision/run identifies a concrete release candidate. ## Runtime Join Contract -- `evidence.path` is the canonical repo-relative proof identity. +- `evidence.path` is the canonical repo-relative evidence identity. - `evidence.test_case` optionally pins one case within the path; matching is trimmed and case-insensitive. - Use the exact `test_case` emitted after native-report conversion. Never derive it from a shortened source-code label when the reporter emits a nested or otherwise transformed identity. -- Apply the same rule to manual, smoke, and agent proof: a checklist heading, +- Apply the same rule to manual, smoke, and agent evidence: a checklist heading, workflow label, or scenario title becomes a pin only when the canonical producer emits that exact identity. Before then it belongs in `notes` or `command`, not `test_case`. @@ -269,7 +269,7 @@ the observed revision/run identifies a concrete release candidate. matches `evidence.path`; `test_case` matches `evidence.test_case`. If one observation matches both a file-level and pinned row, remove the overlap -across all feature maps: keep the proof file-level or pin every distinct row. +across all feature maps: keep the verification method file-level or pin every distinct row. ## Connect an observation source @@ -329,16 +329,16 @@ Follow this sequence. Do not ask the user to choose a parser or config shape. GitHub Actions metadata comes from `GITHUB_SHA`, `GITHUB_REF_NAME`, and `GITHUB_RUN_ID`. Outside GitHub Actions, supply `--commit`; `--branch`, `--run-id`, `--run-url`, and `--observed-at` are optional. -4. **Validate before upload.** +4. **Schema-validate before upload.** ```bash npx --yes @shiplightai/quality-tools@^0.3.0 observations validate \ quality-observations.json ``` Compare the canonical observations with every mapped identity they are meant - to satisfy. Count exact matches, unmatched mapped proof, unmatched + to satisfy. Count exact matches, unmatched mapped methods, unmatched observations, and ambiguous observations from parsed output—not prose - estimates. Do not call the connection complete while intended proof is + estimates. Do not call the connection complete while an intended method is unmatched or ambiguous. 5. **Publish canonical files.** Upload one `quality-observations.json` per @@ -381,12 +381,12 @@ unobserved; never manufacture a missing record. For a CI smoke or health gate, map the workflow file as `evidence.path`, use the step/check name as `evidence.test_case`, and emit those values through -`observations record`. Without the canonical file, record a proof gap rather +`observations record`. Without the canonical file, record an evidence gap rather than pretending the workflow is observed. ## Generate fix prompts -For agent-ready proof-gap prompts: +For agent-ready evidence-gap prompts: ```bash npx --yes @shiplightai/quality-tools@^0.3.0 fix-prompts \ @@ -421,7 +421,7 @@ It must not: - write run outcomes into quality maps - hand-edit generated recommendations or fix prompts - mint a feature slug for project-wide work -- self-ratify structure or accept risk +- self-validate structure or accept risk - include secrets or private customer data in artifacts ## Report @@ -430,7 +430,7 @@ Report: - baseline and final feature scope, observation set, and observed revision/run - root-cause class for each addressed recommendation -- graph, proof, implementation, or wiring changes made +- graph, evidence, implementation, or wiring changes made - verification commands and auditable outcomes - every available score before and after, with the reason for any unavailable one - remaining work split into agent-actionable, external blocker, deferred, and diff --git a/agent-skills/quality/references/map-feature/assets/quality-map.template.yaml b/agent-skills/quality/references/map-feature/assets/quality-map.template.yaml index 71b0c39..3624df6 100644 --- a/agent-skills/quality/references/map-feature/assets/quality-map.template.yaml +++ b/agent-skills/quality/references/map-feature/assets/quality-map.template.yaml @@ -7,12 +7,12 @@ # spec/user_authored = trusted (1.0), agent_generated = medium (0.7), # inferred_brownfield = low (0.4), unspecified = 0 (counted, earns no trust). structure_provenance: "unspecified" # spec | user_authored | agent_generated | inferred_brownfield | unspecified — set honestly; left as the scores-0 default so a copied template never over-claims trust -# Gate 4 (map-level human review): a human reviewed and approved this whole check +# Gate 4 (map-level human review): a human reviewed and validated this whole check # list. Combined with a confirmed feature (gate 2), it lifts the feature's checks to # HIGH structure confidence, overriding the per-check provenance ladder. Orthogonal # to structure_provenance (origin), which review never overwrites. HUMAN-GATED: an # agent may propose the reviewed list but must never set this on the owner's behalf. -checks_reviewed: false # Optional; true only when a human has ratified the check list +checks_reviewed: false # Optional; true only when a human has validated the check list against accepted intent target: id: "001-example-feature" name: "" @@ -28,7 +28,7 @@ expectations: description: "" source_type: "SOURCE" # SOURCE | IMPLEMENTATION | INFERRED # Optional: override the map-level structure_provenance for THIS check only, - # e.g. one hand-verified check in an otherwise inferred_brownfield map. + # e.g. one human-authored check in an otherwise inferred_brownfield map. # structure_provenance: "user_authored" source_refs: - path: "" @@ -37,7 +37,7 @@ expectations: category: "other" # billing | auth | security | data | ui | api | ops | performance | privacy | compliance | other # Declared importance — a FACT read from a dev artifact (PRD, # feature-breakdown, spec, or per-behavior in test-spec.md), never invented - # here. priority is the proof-effort driver (P0 = strongest posture). Its + # here. priority is the verification-effort driver (P0 = strongest posture). Its # trust rides on structure_provenance above. Use UNKNOWN only when no # artifact declares it; do not guess a level and record it as fact. priority: "P1" # P0 | P1 | P2 | P3 | UNKNOWN @@ -50,10 +50,10 @@ expectations: policy_override: preferred_modalities: [] # Optional: unit | contract | integration | e2e | agent | manual | telemetry | static | smoke | script | other discouraged_modalities: [] # Optional: modalities to avoid unless no better option exists - required_modalities: [] # Optional: modalities that must appear in the proof set + required_modalities: [] # Optional: modalities that must appear in the mapped method set required_contexts: [] # Optional: ["local", "pr-ci", "staging-gate"] - require_gate: false # Optional: at least one proof should be suitable for a gate context - notes: "" + require_gate: false # Optional: at least one method should be suitable for a gate context + notes: "" # Accepted risk (HUMAN-GATED): gap categories a human has reviewed and accepted # as tolerated risk for this check. An accepted gap stays visible but stops @@ -63,7 +63,7 @@ expectations: # may PROPOSE an acceptance but must never accept a risk on the owner's behalf. accepted_gaps: [] # Optional: subset of [missing, stale, deferred, manual-only, weak, failing, unavailable] - # A proof definition is just the test TYPE (a fact about the artifact) at a + # A verification method is just the test TYPE (a fact about the artifact) at a # PATH. Evidence confidence is derived downstream from `type` via a # transparent rubric — do not hand-author depth or reliability here. # Flakiness, staleness, and pass/fail come from runtime observations joined @@ -71,15 +71,15 @@ expectations: evidence: - id: "" type: "integration" # unit | contract | integration | e2e | agent | manual | telemetry | static | smoke | script | other - path: "" + path: "" url: "" - command: "" - contexts: ["local"] # Contexts where this proof is intended to run or matter - notes: "" + command: "" + contexts: ["local"] # Contexts where this method is intended to run or matter + notes: "" # Smoke / health check that runs in CI but is not a test file. Point `path` # at the workflow file so a reviewer can open it. Add `test_case` only after - # a canonical observation proves the emitted identity, or when improve wires + # a canonical observation establishes the emitted identity, or when improve wires # that identity at the producer boundary in the same change. A visible job, # step, or manual-check label is never sufficient by itself; keep that # readable pointer in notes or command until canonical emission exists. @@ -93,5 +93,5 @@ expectations: contexts: ["ci"] proof_gap: - summary: "" - next_step: "" + summary: "" + next_step: "" diff --git a/agent-skills/quality/references/map-feature/index.md b/agent-skills/quality/references/map-feature/index.md index 1a8a91b..776c100 100644 --- a/agent-skills/quality/references/map-feature/index.md +++ b/agent-skills/quality/references/map-feature/index.md @@ -1,20 +1,20 @@ -# map-feature — Map one feature's checks and proof +# map-feature — Map one feature's checks and verification methods `map-feature ` constructs or improves one feature's section of the quality graph: ```text -feature → quality checks → proof definitions +feature → quality checks → verification methods ``` It writes the feature's `quality-map.yaml` from accepted requirements and -existing proof artifacts so the engine can score coverage, evidence confidence, +existing evidence artifacts so the engine can score coverage, evidence confidence, and structure confidence. ## Contents - Boundaries, inputs, and graph artifact -- Check and proof authoring rules +- Check and verification-method authoring rules - Structure provenance and human gates - Runtime join contract - Workflow, validation, and edit boundaries @@ -26,11 +26,11 @@ and structure confidence. - [vocabularies](../_shared/vocabularies.md) Work on one feature, spec, module, PR, or ticket at a time. The outcome is a -trustworthy answer to two questions: what must hold for this feature, and what -existing artifact proves each check? +trustworthy answer to two questions: what must hold for this feature, and which +existing verification method evaluates each check? -`map-feature` reads proof facts and connects them to the graph; it does not -generate the proof: +`map-feature` reads verification-method and evidence facts and connects them to +the graph; it does not generate the evidence: - It does **not** create tests or pick testing strategy — that is `/shiplight cover`. This command reads the test-spec, the test-report, and the actual @@ -46,7 +46,7 @@ than one feature's graph, use that command. ## Graph layer and artifact Per target, at `.quality/evidence//quality-map.yaml`: the structural -proof-definition graph — the quality checks (`expectations`), each carrying a +verification-method graph — the quality checks (`expectations`), each carrying a declared `priority`, evidence rows (`type` + `path`), `structure_provenance`, and `proof_gap` guidance. It is structural only: no run outcomes, timestamps, freshness, or confidence rollups (those are observations/evaluations). @@ -65,19 +65,19 @@ testing-what and each behavior's declared priority), `specs//test-repor schemas, routes, and CI, and the PRD/spec for declared priorities. Construction is the same path whether spec-driven or brownfield; only the confidence differs — spec-driven inputs yield high structure confidence, inference from code/tests -alone starts `inferred_brownfield` until a human ratifies it. +alone starts `inferred_brownfield` until a human validates it. This skill **records two facts and derives scores from them; it authors no -judgments**, and it verifies facts rather than copying the dev session's claims: +judgments**, and it confirms facts rather than copying the dev session's claims: - **`priority` (P0–P3)** — read from the declaring artifact (PRD, feature breakdown, spec, or per-behavior test-spec), never invented. Mark `UNKNOWN` when nothing declares it; do not guess. It is the importance signal — there is no 1–5 risk weight — and its trust rides on `structure_provenance`. -- **Evidence `type`** — a fact about the cited proof definition, confirmed from +- **Evidence `type`** — a fact about the cited verification method, confirmed from what it actually executes at `evidence.path` and optional `evidence.test_case`, not copied from its filename, directory, runner, or report label. Classify the - execution boundary: isolated behavior is `unit`; proof that exercises a + execution boundary: isolated behavior is `unit`; evidence from a method that exercises a contract or interaction across real components is `contract` or `integration` even when external infrastructure is simulated. Evidence confidence is **derived from type** by a transparent rubric (manual < single automated < @@ -106,10 +106,10 @@ under `expectations`. Each should include: - Related implementation tasks when available. - Evidence rows: schema-valid `type` at a `path` (+ optional `test_case`, `contexts`), the type confirmed against the artifact. -- Optional `proof_gap` describing what proof is still missing and what to add - next. +- Optional `proof_gap` describing the evidence gap and which verification method + to add next. -Keep the map structural: proof definitions, declared priority, and proof gaps +Keep the map structural: verification methods, declared priority, and evidence gaps belong here; run outcomes and derived judgments do not. Preserve stable ids so downstream observation and evaluation systems can join on them. Copy `assets/quality-map.template.yaml` for new maps and validate with @@ -134,14 +134,14 @@ behavior, browser or CLI execution, deployment wiring, or release gates. ## Evidence Type Consistency Before adding an evidence row, search existing quality maps for the same `path` -and optional `test_case`. Reuse a previously verified type when it describes the -same proof boundary. If an existing mapping conflicts with the artifact, report +and optional `test_case`. Reuse a previously confirmed type when it describes the +same verification boundary. If an existing mapping conflicts with the artifact, report the conflict and use the type justified by inspection; do not silently create a second classification. -A file can contain proofs with different boundaries. Distinguish them with +A file can contain verification methods with different boundaries. Distinguish them with `test_case` and classify each cited case independently. Unpinned file-level -evidence must accurately describe the proof claimed for the file as a whole; +evidence must accurately describe the verification method represented by the file as a whole; do not use an unpinned row to hide mixed execution modes. Different types for one path are valid only when distinct pinned cases actually exercise different boundaries. @@ -149,15 +149,15 @@ boundaries. Before validation, audit the whole project for each evidence path. Choose one identity strategy per path: file-level, or exact pins for every mapped case. Report and resolve mixed pinned/unpinned rows; schema validity alone does not -prove that runtime observations will resolve unambiguously. +confirm that runtime observations will resolve unambiguously. ## Structure Provenance And Structure Confidence Declare `structure_provenance` at the top of `quality-map.yaml` (and optionally per check) so the `quality-tools` engine can report **structure confidence** — how much the map's structure (its set of checks *and their priorities*) can be trusted — as a -separate axis from evidence confidence. Evidence confidence asks "is each check -proven?"; structure confidence asks "is this the right set of checks at the +separate axis from evidence confidence. Evidence confidence asks "does each +check have sufficiently strong evidence?"; structure confidence asks "is this the right set of checks at the right priorities, and where did it come from?". The two are reported side by side and never blended. @@ -195,19 +195,19 @@ Rules: (1.0). The agent may author at `inferred_brownfield` and *propose* checks and priorities, but must not record `user_authored`/`spec` without genuine human authorship or an accepted spec. Origin is not review — an `agent_generated` list a -human has *approved* still reads `agent_generated`; the approval is recorded by +human has *validated* still reads `agent_generated`; the validation is recorded by `checks_reviewed` (below), which the engine treats as the review gate. ### `checks_reviewed` — gate 4 (map-level human review) Set `checks_reviewed: true` at the map level ONLY when a human has reviewed and -approved the whole check list. Combined with a confirmed feature (gate 2), it lifts +validated the whole check list against accepted intent. Combined with a confirmed feature (gate 2), it lifts that feature's checks to **HIGH** structure confidence (1.0), overriding the gate-1 -origin ladder — so a human-approved `agent_generated` list scores HIGH without -rewriting its origin. It is **human-gated**: surface the unratified checks — +origin ladder — so a human-validated `agent_generated` list scores HIGH without +rewriting its origin. It is **human-gated**: surface the unvalidated checks — highest-priority first — for review; *propose* the reviewed list, but never flip `checks_reviewed` to true on the owner's behalf. No test and no `fix-prompts` run can -raise it. (Mapping more proof and stronger types raises coverage and evidence +raise it. (Mapping more verification methods and stronger types raises coverage and evidence confidence, reported beside structure confidence and never substituting for it.) ### `accepted_gaps` — accepted risk (human-gated) @@ -223,19 +223,19 @@ changes evidence confidence. Like the gates, it is **human-gated**: the agent ma the owner. Remove the category to un-accept. `structure_provenance` is **gate 1** and `checks_reviewed` **gate 4** of the four -ratification gates that feed structure confidence; the feature-level gates — feature +validation gates that feed structure confidence; the feature-level gates — feature `status` and `priority_provenance` in `project-map.yaml`—are owned by `map-project`, and the engine joins all four. See [`independence.md`](../_shared/independence.md) → -"Structure confidence: the ratification gates". +"Structure confidence: the validation gates". ## Runtime Join Contract The canonical interface between feature quality maps and observations. -`improve` makes proof producers emit the canonical observation format and +`improve` makes evidence producers emit the canonical observation format and configures sources that locate it; evidence authored here must honor it: -- `evidence.path` is the canonical proof-source identity. Prefer stable +- `evidence.path` is the canonical evidence-artifact identity. Prefer stable repo-relative paths aligned with emitted artifact paths. - `evidence.test_case` is an optional pin within that path. Matching is whitespace-trimmed and case-insensitive. @@ -259,7 +259,7 @@ configures sources that locate it; evidence authored here must honor it: Many release gates are smoke/health checks that run in CI but are not test files. They are valid runtime evidence. Author the map side: set `path` to the workflow file that wires the gate. Add `test_case` only when an existing canonical -observation proves the exact emitted identity, or when `improve` configures that +observation establishes the exact emitted identity, or when `improve` configures that identity at the producer boundary in the same change. A workflow job/step label or manual-check heading is never sufficient by itself. When no canonical observation exists yet, keep the evidence file-level, record the @@ -273,13 +273,13 @@ Readers may present `title`, `description`, `proof_gap.summary`, and `proof_gap.next_step` directly, so write them as user-facing summaries. Use the schema; do not add free-form keys. -- `title`: name the project behavior or quality promise the check proves—not a +- `title`: name the project behavior or quality claim—not a command, artifact, or test file. -- `description`: explain what the check proves and which feature behavior or +- `description`: explain the claim and which feature behavior or release confidence it affects. -- `proof_gap.summary`: describe only the structural proof gap or current +- `proof_gap.summary`: describe only the structural evidence gap or current limitation. Put run history in observation artifacts. -- `proof_gap.next_step`: the highest-value proof to add next, or omit `proof_gap` +- `proof_gap.next_step`: the highest-value verification method to add next, or omit `proof_gap` if there is no open gap. Keep `SOURCE` accepted promises, `IMPLEMENTATION`-observed checks, and `INFERRED` @@ -298,16 +298,16 @@ check compact and secondary — do not let it carry the feature's coverage story `priority` (read, not invented — mark `UNKNOWN` if undeclared) and set `source_type`. 4. **Map evidence.** For each check, add evidence rows of `type` + `path`, - confirming the type against the cited proof boundary and checking existing + confirming the type against the cited verification boundary and checking existing maps for conflicting classifications. Honor the Runtime Join Contract. For every existing or proposed `test_case`, locate and cite the canonical observation record containing the same `path` + `test_case`. If none exists, remove the pin, preserve its human-readable label in `notes` or `command`, - and record the emission gap. A matching label in the proof source is not a - substitute. Record `proof_gap` where proof is missing or weak. -5. **Set provenance.** Set `structure_provenance` honestly. Surface unratified, - highest-priority checks for human ratification. -6. **Validate.** Run + and record the emission gap. A matching label in the evidence source is not a + substitute. Record `proof_gap` where evidence is missing or weak. +5. **Set provenance.** Set `structure_provenance` honestly. Surface unvalidated, + highest-priority checks for human validation. +6. **Schema-validate.** Run `npx --yes @shiplightai/quality-tools@^0.3.0 validate `. It runs the engine's real validator—unknown-field, required-field, duplicate-id, source-ref, and evidence-path checks—and exits non-zero on any error @@ -319,14 +319,14 @@ check compact and secondary — do not let it carry the feature's coverage story ``` Keep the map structural—no run outcomes or rollups. - When a native report can be produced without changing proof semantics, + When a native report can be produced without changing verification semantics, convert a representative report to canonical observations and verify every new `path` + `test_case` identity against it. Remove any pin that cannot be supported by a canonical record; report its runtime-emission gap rather than claiming the map is fully connected. 7. **Optional runtime hand-off.** When the user wants runtime results connected, use `improve` for `.quality/config/*`, then `assess`. This command contributes - the map side: evidence `path`/`test_case` and proof gaps. + the map side: evidence `path`/`test_case` and evidence gaps. ## Artifact Skeletons @@ -349,7 +349,7 @@ cover`; for `.quality/config/*` see `improve`. - Never author `depth`, `reliability`, a risk weight, or a `HIGH/MEDIUM/LOW` verdict; `priority` and evidence `type` are read/confirmed facts (see Inputs, Facts, And Independence), never invented. -- Never self-promote `structure_provenance`; ratification is human-gated. +- Never self-promote `structure_provenance`; validation is human-gated. - Keep `quality-map.yaml` structural: preserve ids, use schema enums, no run state, timestamps, freshness, or confidence rollups. - Never report pass/fail without command output or an auditable observation. diff --git a/agent-skills/quality/references/map-project/assets/project-map.template.yaml b/agent-skills/quality/references/map-project/assets/project-map.template.yaml index 3f8c71e..b67f04a 100644 --- a/agent-skills/quality/references/map-project/assets/project-map.template.yaml +++ b/agent-skills/quality/references/map-project/assets/project-map.template.yaml @@ -3,7 +3,7 @@ project: id: "" name: "" summary: "" - # Reference to the dev-owned proof-strategy artifact, when one exists. + # Reference to the dev-owned verification-strategy artifact, when one exists. quality_policy_path: "TESTING.md" source_refs: - path: "README.md" @@ -40,7 +40,7 @@ features: - id: "001-example-feature" name: "Example Feature" description: "" - status: "candidate" # gate 2: keep candidate while agent-proposed. After human ratification, use the accurate lifecycle status, such as planned, specified, or implemented. + status: "candidate" # gate 2: keep candidate while agent-proposed. After human validation, use the accurate lifecycle status, such as planned, specified, or implemented. priority: "P1" priority_provenance: "agent" # gate 3: agent | human. Set to human once a person sets/adjusts priority; the agent must not overwrite a human-set priority on rebuild. source_type: "INFERRED" diff --git a/agent-skills/quality/references/map-project/brownfield-reconstruction.md b/agent-skills/quality/references/map-project/brownfield-reconstruction.md index 44363cb..8ceb85f 100644 --- a/agent-skills/quality/references/map-project/brownfield-reconstruction.md +++ b/agent-skills/quality/references/map-project/brownfield-reconstruction.md @@ -13,20 +13,20 @@ Use this workflow when accepted intent or feature history is incomplete. ## Principle Existing code is observed behavior, not automatically intended behavior. The goal is -to build a provisional project graph, then ask the user to ratify it before +to build a provisional project graph, then ask the user to validate it before turning observations into authoritative specs. If the repo already has specs but code changed outside the spec process, treat that as drift, not as automatic truth. Identify the conflict and ask the user to -ratify whether the code behavior should update the spec or be changed back to -the spec. +validate which behavior represents product intent: update the spec to match the +code or change the code back to the spec. ## Operating Posture Brownfield works best as a collaboration, not autonomous reconstruction: - **User drives**: supplies intent sources (PRD/design-doc paths, tracker - queries), sets scope and priority (where to start), and ratifies candidates. + queries), sets scope and priority (where to start), and validates candidates. - **Agent ingests**: reads docs, code, tests, and trackers; proposes candidate features; maps evidence; surfaces conflicts and open questions. @@ -75,7 +75,7 @@ Prefer `rg` and targeted file reads. Avoid broad context dumps. 3. **Assign provisional IDs** - Use existing numeric specs if present. - Otherwise assign stable `001-*`, `002-*` IDs in dependency order. - - Mark status `candidate` until ratified. + - Mark status `candidate` until validated. 4. **Record source types** - `SOURCE` for endorsed intent from PRDs, design docs, or accepted tracker @@ -91,7 +91,7 @@ Prefer `rg` and targeted file reads. Avoid broad context dumps. - Include a user-facing project boundary and feature summaries plus intent docs, code refs, test refs, open questions, and orphan areas. -6. **Ask for ratification** +6. **Ask for validation** - Present the feature list with source type and open questions. - Ask the user to accept, split, merge, rename, defer, or reject features. @@ -126,9 +126,9 @@ For each candidate, capture: - source type - whether behavior seems current, legacy, or deprecated -## User Ratification Prompt Shape +## User Validation Prompt Shape -When asking the user to ratify, keep it concrete: +When asking the user to validate intent, keep it concrete: ```text I found these candidate features: diff --git a/agent-skills/quality/references/map-project/index.md b/agent-skills/quality/references/map-project/index.md index fd76ec2..d2c90bb 100644 --- a/agent-skills/quality/references/map-project/index.md +++ b/agent-skills/quality/references/map-project/index.md @@ -7,7 +7,7 @@ project → features → dependencies ``` It writes `.quality/project-map.yaml`. It does not define a feature's quality -checks or proof; that is `map-feature`. +checks or verification methods; that is `map-feature`. ## Contents @@ -33,7 +33,7 @@ Produce a user-facing map that lets a reader answer: - Which reusable features make up that project? - Which features and priorities are accepted intent versus agent proposals? -- Where are each feature's spec, implementation, proof report, and quality map? +- Where are each feature's spec, implementation, evidence report, and quality map? Use `assets/project-map.template.yaml` as the source of truth for shape. Define reusable feature subsets in `.quality/config/views.yaml`, not in a @@ -46,11 +46,12 @@ parallel project-map grouping. - write a PRD, feature spec, plan, or tasks - drive development or switch branches - create tests or choose testing strategy -- construct per-feature checks/proof +- construct per-feature checks or verification methods - connect runtime observations or compute scores -Use the relevant development workflow for intent/spec work, a proof producer -for test creation, `map-feature` for one feature's checks/proof, and `assess` for +Use the relevant development workflow for intent/spec work, an evidence producer +for test creation, `map-feature` for one feature's checks and verification +methods, and `assess` for scores. ## Choose the construction source @@ -85,7 +86,7 @@ intended project behavior: Reconcile rather than rebuild: - preserve stable ids -- preserve human-set priority and ratification fields +- preserve human-set priority and validation fields - add missing intent links - surface conflicts and drift - never replace accepted structure with a fresh inference pass @@ -112,7 +113,7 @@ Reconcile rather than rebuild: - Define features as independently checkable capabilities, not filesystem directories, packages, or temporary changes. - Give each feature a stable id, promise, lifecycle status, priority, - dependencies, source type, artifact paths, code refs, proof refs, open + dependencies, source type, artifact paths, code refs, evidence refs, open questions, and residual risks. - Record cross-feature concerns explicitly. @@ -149,7 +150,7 @@ Reconcile rather than rebuild: such as `planned`, `specified`, or `implemented`. - Set `priority_provenance: human` only when a human set or confirmed the priority. - - Never ratify on the owner's behalf. + - Never validate intent on the owner's behalf. 8. **Choose the next feature** - Recommend `map-feature ` for the highest-priority accepted or @@ -173,7 +174,7 @@ truth. - accepted specs without implementation - implementation behavior absent from accepted specs - tests asserting behavior absent from accepted specs -- features with no quality map or proof +- features with no quality map or mapped verification methods - stale reports - unresolved cross-feature concerns @@ -184,7 +185,7 @@ resolves intent and implementation drift. `map-project` owns: -- feature `status`: `candidate` remains unratified +- feature `status`: `candidate` remains unvalidated - `priority_provenance`: `human` only after a human priority decision `map-feature` owns `structure_provenance` and `checks_reviewed`. The engine joins @@ -207,7 +208,7 @@ Report: ## When another command is better - No graph exists and the user wants guided setup: `start` -- One feature needs checks/proof mapped: `map-feature` +- One feature needs checks or verification methods mapped: `map-feature` - Existing graph needs scores refreshed: `assess` - Current scores/gaps should be acted on: `improve` - User only wants current state: `status` diff --git a/agent-skills/quality/references/map-project/project-map.md b/agent-skills/quality/references/map-project/project-map.md index 3629b71..5d738d2 100644 --- a/agent-skills/quality/references/map-project/project-map.md +++ b/agent-skills/quality/references/map-project/project-map.md @@ -1,7 +1,7 @@ # Project Map Reference Use the project map as the root of one quality project. It connects accepted -intent, reusable features, implementation artifacts, and proof references. It is +intent, reusable features, implementation artifacts, and evidence references. It is an index and traceability graph, not a replacement for PRDs, specs, policies, tasks, code, feature quality maps, or saved assessment scopes. @@ -40,11 +40,11 @@ Maintain this authority model: ```text accepted PRD / roadmap / user decision: project intent .quality/project-map.yaml: project and feature graph -TESTING.md: proof-strategy guidance (development-owned) +TESTING.md: verification-strategy guidance (development-owned) spec.md: accepted feature behavior plan.md / tasks.md: execution contract code: implementation artifact -tests / reports / reviews: proof +tests / reports / reviews: evidence ``` Specs are current snapshots of accepted behavior, not changelogs. Git records @@ -73,7 +73,7 @@ project-specific fields only when they have a clear consumer. Important fields: - `project`: identity, user-facing purpose, assessment boundary, source refs, - and proof-policy path + and verification-policy path - `product_docs`: accepted requirements, feature catalogs, architecture docs, roadmaps, and other intent sources - `feature_order`: top-level display order for the project index; ordering only, @@ -126,7 +126,7 @@ Include: - `dependencies` - links to accepted intent, spec, plan, tasks, and checklists - code references owned or primarily touched by the feature -- quality-map and proof-report paths +- quality-map and evidence-report paths - open questions and residual risks A feature should be independently specifiable and checkable. Do not create one @@ -145,14 +145,14 @@ performed the mapping. Recommended feature statuses: -- `candidate`: proposed but not human-ratified +- `candidate`: proposed but not human-validated - `planned`: accepted future capability - `specified`: accepted spec exists and requirements are mostly clear - `designed`: plan/contracts/data model exist - `tasked`: implementation tasks exist - `implementing`: code work in progress - `implemented`: code complete for accepted scope -- `verified`: relevant proof has passed +- `verified`: relevant verification methods have passing results - `reviewed`: code review completed without blocking findings - `done`: feature accepted for the current project scope - `blocked`: cannot progress without user or external state @@ -192,8 +192,8 @@ Surface: - accepted specs with no implementation - implementation behavior missing from accepted specs - tests asserting behavior absent from accepted specs -- features with no quality map or proof -- stale proof reports +- features with no quality map or mapped verification methods +- stale evidence reports - changed code without updated specs - unresolved cross-feature concerns diff --git a/agent-skills/quality/references/start.md b/agent-skills/quality/references/start.md index eab451f..1634fd8 100644 --- a/agent-skills/quality/references/start.md +++ b/agent-skills/quality/references/start.md @@ -26,7 +26,7 @@ plus an initial assessment when runtime observations are available. Before discussing YAML, explain: ```text -project → features → quality checks → proof definitions → runtime observations +project → features → quality checks → verification methods → runtime observations ``` Then show which existing repository artifacts can supply each layer. Do not ask @@ -39,7 +39,7 @@ config before explaining their roles. accepted requirements as project intent; do not infer them again from code. - **Brownfield:** current code/tests/docs exist without accepted feature structure. Reconstruct candidates from implementation and ask the user to - ratify them. + validate them against product intent. - **Partial graph:** some `.quality/` artifacts exist. Preserve stable ids and human-owned fields, repair only the missing or stale layers, and never replace the graph wholesale. @@ -55,14 +55,14 @@ start. Do not autonomously map an entire brownfield repository. ## Bootstrap workflow 1. **Inventory without mutation** - - Read accepted intent/spec documents, `.quality/**`, proof policy such as + - Read accepted intent/spec documents, `.quality/**`, verification policy such as `TESTING.md`, relevant code/test areas, CI workflows, and recent reports. - Report which graph layers exist, are missing, or conflict. 2. **Choose the first feature scope** - Prefer the user-named area. - Otherwise use the highest declared feature priority or project risk. - - Keep the initial slice small enough to review and ratify. + - Keep the initial slice small enough to review and validate. 3. **Map the project** - Follow `map-project`. @@ -86,8 +86,10 @@ start. Do not autonomously map an entire brownfield repository. - Follow `map-feature `. - Construct checks from accepted requirements when they exist; otherwise reconstruct them from implementation with honest provenance. - - Connect existing proof by canonical path and optional test-case name. - - Record missing or weak proof as `proof_gap`; do not create tests here. + - Connect existing verification methods by canonical path and optional + test-case name. + - Record missing or weak evidence in the literal `proof_gap` field; do not + create tests here. - Set `structure_provenance: spec` when the check list and priorities trace to accepted requirements. Agent authorship of the YAML does not make that list `agent_generated`; use per-check provenance for genuine exceptions. @@ -128,8 +130,8 @@ Report: - starting-state classification and scope - graph layers created, preserved, or still missing -- proposed versus human-ratified structure -- first feature mapped and proof gaps found +- proposed versus human-validated structure +- first feature mapped and evidence gaps found - saved assessment scopes created, when any - whether runtime assessment was possible - all four engine-produced scores, when available diff --git a/agent-skills/quality/references/status.md b/agent-skills/quality/references/status.md index 46fe298..ed6c210 100644 --- a/agent-skills/quality/references/status.md +++ b/agent-skills/quality/references/status.md @@ -1,7 +1,8 @@ # status — Read the current quality posture `status` is read-only. It may use repository-inspection tools, but it does not -edit files, run proof, invoke `quality-tools`, or refresh generated results. +edit files, execute verification methods, invoke `quality-tools`, or refresh +generated results. ## Read first @@ -18,7 +19,7 @@ edit files, run proof, invoke `quality-tools`, or refresh generated results. - referenced recent test reports when needed to explain staleness 2. Describe the graph in user-facing project language: - features represented and missing - - checks and proof connected for the requested scope + - checks and verification methods connected for the requested scope - views available and the feature ids each selects - observation sources and sets available - unresolved graph edges, drift, or human gates @@ -28,7 +29,8 @@ edit files, run proof, invoke `quality-tools`, or refresh generated results. 4. Recommend exactly one next command: - `start` when no usable graph exists - `map-project` when project/feature structure is missing or stale - - `map-feature ` when one feature lacks trustworthy checks or proof + - `map-feature ` when one feature lacks trustworthy checks or + verification methods - `assess` when the graph is ready but scores need refreshing - `improve` when a current assessment identifies actionable gaps 5. Present the command menu. diff --git a/agent-skills/spec-project/README.md b/agent-skills/spec-project/README.md index f371294..de4dd68 100644 --- a/agent-skills/spec-project/README.md +++ b/agent-skills/spec-project/README.md @@ -31,7 +31,7 @@ PRD -> feature breakdown -> accepted feature spec -> plan and tasks ``` The feature spec is the authority for product behavior. Code implements that -behavior; tests, verification, and reports prove it. When those artifacts +behavior; tests, verification, and reports provide evidence for it. When those artifacts disagree, `spec-project` reconciles the drift instead of treating the newest artifact as automatically correct. @@ -95,7 +95,7 @@ maintenance: The skill first identifies every affected existing feature, updates the accepted product behavior, and then coordinates the smallest implementation -and proof changes. +and evidence changes. To inspect a project without changing it, invoke the skill with no operation: @@ -166,7 +166,7 @@ handoff. - **Implementation complete:** the code implements that accepted scope and the relevant implementation checks pass. - **Evidence complete:** `/shiplight cover` has reconciled `test-spec.md`, run - the relevant proof, and recorded it in `test-report.md`. + the relevant verification methods, and recorded them in `test-report.md`. For a full lifecycle request, the feature is not done until all three states are complete. Missing tools, blocked checks, and remaining risks are reported diff --git a/agent-skills/spec-project/SKILL.md b/agent-skills/spec-project/SKILL.md index 8c470a3..5bed44f 100644 --- a/agent-skills/spec-project/SKILL.md +++ b/agent-skills/spec-project/SKILL.md @@ -7,7 +7,7 @@ description: Drive specification-led project development from product intent thr Drive project work from accepted product intent to implementation and durable testing evidence. Treat the artifacts as interfaces that keep intent, -implementation, and proof aligned; do not treat document production itself as +implementation, and evidence aligned; do not treat document production itself as the outcome. ## Delegation Contract @@ -22,7 +22,7 @@ ownership of artifacts produced by a lower-level workflow. | `specs/NNN-feature-name/spec.md` | Spec Kit in Spec Kit mode; portable coding agent otherwise: accepted product behavior | | `specs/NNN-feature-name/plan.md` | Spec Kit in Spec Kit mode; portable coding agent otherwise: implementation approach | | `specs/NNN-feature-name/tasks.md` | Spec Kit in Spec Kit mode; portable coding agent otherwise: executable work | -| `specs/NNN-feature-name/test-spec.md` | `/shiplight cover`: what must be proved and at what declared priority | +| `specs/NNN-feature-name/test-spec.md` | `/shiplight cover`: what must be verified and at what declared priority | | `specs/NNN-feature-name/test-report.md` | `/shiplight cover`: tests, commands, and observed results | | Tests and verification artifacts | `/shiplight cover` and its producers | @@ -34,7 +34,7 @@ artifacts. - Treat `spec.md` as the product-behavior authority in both operating modes. - Treat code as an implementation of the product spec. -- Treat `test-spec.md`, tests, reports, verification, and reviews as proof +- Treat `test-spec.md`, tests, reports, verification, and reviews as evidence artifacts, not substitutes for product intent. - Keep every `spec.md` as the latest accepted product snapshot, never a chronological change log. A reader must be able to determine all current @@ -51,7 +51,7 @@ When behavior changes, have the active feature-spec writer reconcile the whole `spec.md` into that current snapshot and obtain acceptance before changing the implementation. Do not satisfy this gate by appending the change request to the old spec. Then delegate implementation through the active mode and ask -`/shiplight cover` to reconcile its testing contract and proof. If product +`/shiplight cover` to reconcile its testing contract and evidence. If product intent is ambiguous or artifacts conflict materially, stop and ask the owner; never silently choose product semantics. @@ -96,7 +96,7 @@ status pass. ## Reconcile Before Creating Treat every incoming product change as an existing-feature change until the -repository proves otherwise. Before assigning a feature ID, creating a feature +repository evidence shows otherwise. Before assigning a feature ID, creating a feature directory, or invoking a new-feature Spec Kit workflow: 1. Read the PRD and feature breakdown. Inventory every existing @@ -152,7 +152,7 @@ capability. Complete the reconciliation gate, then select an existing feature whenever it can own the change. Resolve a new feature ID and target only after reporting why the inspected existing features do not apply. In portable mode, let the coding -agent create the directory only after that proof; do not create or update Spec +agent create the directory only after that evidence; do not create or update Spec Kit pointers. In Spec Kit mode, invoke its new-feature workflow only after the same gate, then let Spec Kit create the feature directory and maintain its pointers. @@ -181,7 +181,7 @@ Then run the selected feature through these gates: 3. **Plan and task:** in Spec Kit mode, delegate `plan.md` and `tasks.md` to its plan and tasks workflows. In portable mode, direct the coding agent through existing project conventions. Keep tasks traceable to accepted requirements. -4. **Plan proof:** hand the accepted spec and declared priorities to +4. **Plan verification:** hand the accepted spec and declared priorities to `/shiplight cover`. Ask it to create `test-spec.md` and tests before production changes when a regression-first or test-first sequence is practical. If it is not practical, record why and invoke it after the @@ -192,8 +192,8 @@ Then run the selected feature through these gates: keep spec, plan, and tasks current as implementation facts emerge; ask the owner before changing product semantics. 6. **Verify:** use `/shiplight verify` for relevant live UI/API behavior. -7. **Complete proof:** invoke `/shiplight cover` to refresh the testing contract, - run the relevant proof, and write `test-report.md`. Optionally invoke +7. **Complete verification:** invoke `/shiplight cover` to refresh the testing contract, + run the relevant verification methods, and write `test-report.md`. Optionally invoke `/shiplight review` when the user requests review or the accepted plan calls for it. 8. **Reconcile:** confirm `spec.md` states only the latest accepted behavior and @@ -219,7 +219,7 @@ Use `batch` only to orchestrate preparation of multiple feature specs and plans with the owner available; do not implement multiple features concurrently. Use `autonomous` only for accepted specs whose plans and tasks contain no unresolved product decisions. Work in dependency order and stop on ambiguity, missing -prerequisite work, or a proof decision requiring owner judgment. +prerequisite work, or a verification decision requiring owner judgment. ## Completion States @@ -231,7 +231,8 @@ Report each state separately: - **Implementation complete:** code implements the accepted scope and relevant implementation checks pass. - **Evidence complete:** `/shiplight cover` produced a reconciled - `test-spec.md`, relevant proof passed, and `test-report.md` records the run. + `test-spec.md`, relevant verification methods passed, and `test-report.md` + records the run. For a full lifecycle request, do not report the feature done until all three states are complete. Report unavailable tooling, blocked checks, and residual diff --git a/agent-skills/spec-project/assets/feature-breakdown-template.md b/agent-skills/spec-project/assets/feature-breakdown-template.md index 46817b9..3dae32d 100644 --- a/agent-skills/spec-project/assets/feature-breakdown-template.md +++ b/agent-skills/spec-project/assets/feature-breakdown-template.md @@ -14,7 +14,7 @@ - Replaced behavior is removed from active specs after the replacement is accepted. - Treat each change as an existing-feature retrofit until the existing roadmap - and feature specs prove it introduces an independent product capability. + and feature specs show it introduces an independent product capability. - Do not add a feature entry merely because a requirement is new, a change is large, or a ticket calls it a feature. - Before adding a feature entry, record which existing features were checked and diff --git a/agent-skills/spec-project/references/spec-kit.md b/agent-skills/spec-project/references/spec-kit.md index 4ecb890..e84acd0 100644 --- a/agent-skills/spec-project/references/spec-kit.md +++ b/agent-skills/spec-project/references/spec-kit.md @@ -57,7 +57,7 @@ integration instead of assuming one spelling. After planning is accepted, return to the main lifecycle for the regression-first `/shiplight cover` handoff, implementation, verification, and -final proof run. +final verification run. ## Pointers and Branches diff --git a/docs/README.md b/docs/README.md index d5a2da2..892e0c8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,8 +1,9 @@ # Shiplight Quality guide -Shiplight Quality connects the expected behaviors your product requires to the proof that -supports them. It shows what is covered, what is uncertain, what happened in the -latest test runs, and which decisions still need a person. +Shiplight Quality helps people validate that expected behaviors represent +product intent, then connects those behaviors to verification methods and the +evidence those methods produce. It shows what is covered, what is uncertain, +what happened in the latest runs, and which decisions still need a person. You do not need to understand the YAML files or scoring formulas to use this guide. Most tasks can be requested from a coding agent in plain language. The @@ -18,9 +19,10 @@ It explains the model in a few minutes. Then follow | Page | What you will learn | | --- | --- | -| [The quality graph](concepts/quality-graph.md) | How expected behaviors connect to proof and test results | +| [Correctness terminology](concepts/terminology.md) | How validation, verification, evidence, observations, and proof differ | +| [The quality graph](concepts/quality-graph.md) | How expected behaviors connect to verification methods and results | | [The four scores](concepts/the-four-scores.md) | What each score means and why the scores stay separate | -| [How Quality earns your trust](concepts/trust-boundaries.md) | The boundaries that keep proof, scores, and human decisions independent | +| [How Quality earns your trust](concepts/trust-boundaries.md) | The boundaries that keep evidence, scores, and human decisions independent | | [Who decides what](concepts/who-decides-what.md) | Which work an agent can prepare and which decisions require you | | [Glossary](concepts/glossary.md) | The terms used throughout Quality | @@ -31,9 +33,9 @@ It explains the model in a few minutes. Then follow | Start using Quality in a repository | [Set up Quality](how-to/set-up-quality.md) | | Use requirements from Jira, Linear, or an external document | [Add product sources](how-to/add-product-sources.md) | | Define what one feature must guarantee | [Map a feature](how-to/map-a-feature.md) | -| Review and approve an agent's proposal | [Review and approve the graph](how-to/review-and-ratify.md) | +| Validate an agent's proposal against product intent | [Validate the graph](how-to/validate-the-graph.md) | | Connect CI test results | [Make CI results count](how-to/make-ci-results-count.md) | -| Require stronger proof for an important expected behavior | [Require stronger proof](how-to/require-stronger-proof.md) | +| Require stronger evidence for an important expected behavior | [Set verification requirements](how-to/set-verification-requirements.md) | | Record a risk we have chosen to accept | [Accept a known gap](how-to/accept-a-known-gap.md) | | Assess only the features in one release | [Scope an assessment](how-to/scope-an-assessment.md) | | Understand and improve a low score | [Act on a low score](how-to/act-on-a-weak-score.md) | @@ -53,7 +55,7 @@ the prompts to your project; they are examples, not special syntax. Changes to `.quality/` are normal repository changes. Review them in a pull request just as you would review product requirements or CI configuration. An agent may propose a feature, priority, check, or accepted risk, but it must not -record your approval unless you explicitly give it. +record your validation or approval unless you explicitly give it. ## For contributors diff --git a/docs/commands.md b/docs/commands.md index df9ddcb..56d36b5 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -14,12 +14,12 @@ same outcome in your own words. | `/quality start` | Set up the smallest useful quality graph | [Set up Quality](how-to/set-up-quality.md) | | `/quality status` | Describe the current graph and saved results without changing anything | — | | `/quality map-project` | Create or update the project and feature list | — | -| `/quality map-feature ` | Define one feature's checks and connect existing proof | [Map a feature](how-to/map-a-feature.md) | +| `/quality map-feature ` | Define one feature's checks and connect existing verification methods | [Map a feature](how-to/map-a-feature.md) | | `/quality assess` | Refresh the available scores and explain them | [The four scores](concepts/the-four-scores.md) | | `/quality improve` | Diagnose a weak score or gap and address its cause | [Act on a low score](how-to/act-on-a-weak-score.md) | | `/quality help` | Explain a Quality request without running it | — | -There is no shortcut that approves a proposal or accepts risk. State those +There is no shortcut that validates a proposal or accepts risk. State those decisions in your own words so the agent can record exactly what you decided. See [Who decides what](concepts/who-decides-what.md). @@ -38,7 +38,7 @@ command to see all options. | --- | --- | | `validate` | Validates one feature quality map and exits with an error code when the map is invalid | | `analyze` | Calculates the available scores, optionally loads a saved observation set, and writes the result and any ranked recommendations to JSON | -| `fix-prompts` | Turns structural proof gaps into instructions for a coding agent | +| `fix-prompts` | Turns structural evidence gaps into instructions for a coding agent | | `observations` | Converts JUnit or Playwright reports, records individual outcomes, merges files, validates them, or prints their schema | | `schema` | Prints the quality-map JSON Schema | @@ -46,8 +46,8 @@ command to see all options. **`validate` checks the map contract, not the whole repository.** It checks YAML, required and unknown fields, identifiers, references, and canonical evidence -paths. It does not check whether a referenced proof file currently exists. A -full project scan reports missing evidence files separately. +paths. It does not check whether a referenced verification artifact currently +exists. A full project scan reports missing evidence files separately. **`analyze` needs an observation-set identifier only for the Quality score.** It scans the repository, optionally applies a saved view, and writes a file under diff --git a/docs/concepts/glossary.md b/docs/concepts/glossary.md index de16879..20d836e 100644 --- a/docs/concepts/glossary.md +++ b/docs/concepts/glossary.md @@ -23,15 +23,39 @@ descriptions help people understand the graph but do not change scoring. never charged twice for one order.” Quality map files call checks *expectations*. -**Proof** — An artifact that can support a check: a test, workflow gate, -monitoring signal, static check, or manual procedure. Quality map files call -proof *evidence*. +**Validation** — Comparing proposed features and checks with product intent to +confirm that they describe the right requirements and expected behaviors. -**Observation** — A recorded outcome from one piece of proof: passed, failed, -errored, or skipped. +**Verification** — Evaluating whether the implementation or observed behavior +satisfies a quality check. Verification may be reasoning-based or empirical. -**Proof gap** — A recorded explanation that proof is missing, incomplete, or -not yet available, often with a suggested next step. +**Verification method** — A method or artifact used to evaluate a check, such +as a test, formal analysis, workflow gate, monitoring signal, static check, or +manual procedure. Quality map files store these mappings under `evidence`. + +**Evidence** — Information that supports or challenges a check. Evidence may +come from reasoning or from an observation evaluated against an expected +outcome. + +**Observation** — A recorded fact or outcome. An observation becomes +verification evidence only when it is connected to a check and evaluated +against the expected outcome. + +**Empirical observation** — A recorded fact about executed or production +behavior that empirical verification compares with a quality check. + +**Runtime observation** — Quality's standardized record that an executable +verification method passed, failed, errored, or was skipped at a particular +revision and time. It may transport the outcome of either an empirical method +or an executable reasoning tool. + +**Evidence gap** — A recorded explanation that evidence is missing, incomplete, +or not yet available, often with a suggested next step. The current quality-map +field is named `proof_gap` for compatibility. + +**Proof** — A deductive argument that establishes a proposition under stated +assumptions, including a machine-checked formal proof. Quality does not use +*proof* as a general synonym for tests or evidence. ## Scope and timing @@ -52,36 +76,40 @@ by an assessment. ## Trust and decisions -**Approve or ratify** — For a person to confirm an agent proposal. Human review -can lift structure confidence when the feature is confirmed and its complete -list of checks is approved. +**Validate intent** — For a person to confirm that proposed features and checks +accurately represent the product intent. Human validation can lift structure +confidence when the feature and its complete list of checks are confirmed. **Provenance** — The recorded origin of a list of checks or an individual check: a specification, a person, an agent draft, an inference from existing code, or no declared source. Provenance affects structure confidence and is not overwritten -when someone later approves the checks. +when someone later validates the checks. -**Proof policy** — A requirement that one check needs particular proof, a -specific runtime context, or a delivery gate. An unmet policy can lower coverage, -evidence confidence, and the static quality readout; it does not rewrite a -runtime result that already occurred. +**Verification policy** — A requirement that one check needs a particular +verification method, runtime context, or delivery gate. An unmet policy can +lower coverage, evidence confidence, and the static quality readout; it does not +rewrite a runtime result that already occurred. **Accepted risk** — A gap category that a person has explicitly chosen to tolerate. The gap remains visible but no longer counts as open. Accepting a `missing`, `manual-only`, or `weak` gap also removes that structural scoring penalty. Other accepted categories change gap reporting only. -## Kinds of proof +## Kinds of verification method -Quality recognizes these proof types: +The current quality-map schema recognizes these `evidence.type` values: `unit`, `contract`, `integration`, `e2e`, `agent`, `manual`, `telemetry`, `static`, `smoke`, `script`, and `other`. -The type describes what the artifact actually is. The engine uses that fact, -along with runtime context and declared proof requirements, to derive evidence -confidence. Priority—not proof type—determines how heavily a check counts in an -aggregate score. +The type describes what the mapped method or artifact actually is. The engine +uses that fact, along with runtime context and declared verification +requirements, to derive evidence confidence. Priority—not method type—determines +how heavily a check counts in an aggregate score. + +These values are a compatibility vocabulary rather than a complete taxonomy of +correctness methods. See [Correctness terminology](terminology.md) for the +reasoning and empirical branches. ## Kinds of gap @@ -92,9 +120,9 @@ Open gaps use these categories: The distinction matters: -- **Missing** means no proof is mapped. -- **Unavailable** means proof is described but does not provide a usable path, - URL, or command. A mapped proof item with no matching runtime result is +- **Missing** means no verification method is mapped. +- **Unavailable** means a method is described but does not provide a usable + path, URL, or command. A mapped method with no matching runtime result is **unobserved**, which is a runtime state rather than a gap category. - **Failing** means the available result reported a failure. diff --git a/docs/concepts/quality-graph.md b/docs/concepts/quality-graph.md index b6c9f03..b9a75df 100644 --- a/docs/concepts/quality-graph.md +++ b/docs/concepts/quality-graph.md @@ -10,9 +10,10 @@ you whether an important expected behavior has no test, whether a result failed to reach the reporting system, or whether the team agrees that the right things are being tested. -Quality starts with expected behaviors. It then connects each behavior to proof -and, when available, to a runtime result. Missing links stay visible as gaps -even when every existing test is green. +Quality starts by mapping product intent into features and expected behaviors +that a person can validate. It then connects each behavior to verification +methods and, when available, to observations or other results. Missing links +stay visible as evidence gaps even when every existing test is green. ## The five layers @@ -20,8 +21,8 @@ even when every existing test is green. project what you are assessing └── features the capabilities it provides └── checks the expected behaviors each capability must keep - └── proof tests and other artifacts that can support a behavior - └── results what happened when that proof ran + └── verification methods how each behavior is evaluated + └── runtime observations what executable methods reported ``` Read the graph from top to bottom: @@ -32,14 +33,22 @@ Read the graph from top to bottom: project? “Checkout” is a feature; the `checkout/` folder is not. 3. **Check:** What must remain true about that feature? For example, “A customer is never charged twice for one order.” -4. **Proof:** What could demonstrate that the check holds? This may be a unit - test, browser test, release gate, monitoring signal, or manual procedure. -5. **Result:** What did the selected run report for that proof? - -Each layer can have a different problem. A check may have no proof. Proof may -exist but have no matching result. A passing result may support a list of checks -that nobody has reviewed. Keeping those cases separate makes the next action -clearer. +4. **Verification method:** How can we evaluate whether the check holds? This + may be reasoning-based, such as static analysis, or empirical, such as a unit + test, browser test, monitoring signal, or manual runtime procedure. +5. **Runtime observation:** What did an executable method report? This + standardized result record can carry an empirical observation or the status + of an executable reasoning tool. + +Each layer can have a different problem. A check may have no verification +method. A method may exist but have no matching evidence or result. A passing +observation may support a list of checks that nobody has validated against +product intent. Keeping those cases separate makes the next action clearer. + +Validation and verification answer different questions. Validation connects +product intent to the feature and check structure. Verification connects a +check to the implementation or observed behavior. See +[Correctness terminology](terminology.md) for the complete distinction. ## Where the graph is stored @@ -49,14 +58,16 @@ reviewed and versioned with the rest of the project. | Information | Location | | --- | --- | | Project and feature list | `.quality/project-map.yaml` | -| Checks and proof for one feature | `.quality/evidence//quality-map.yaml` | +| Checks and verification methods for one feature | `.quality/evidence//quality-map.yaml` | | Saved scopes, result sources, and source lists | `.quality/config/` | -Tests, reports, workflows, and other proof stay in their existing locations. -The quality maps point to them; Quality does not move or rewrite them. +Tests, analyses, reports, workflows, and other verification artifacts stay in +their existing locations. The quality maps point to them; Quality does not move +or rewrite them. You do not need to create these files by hand. A coding agent can prepare them, -but you remain responsible for reviewing product decisions and approvals. +but you remain responsible for validating product intent and approving decisions +such as accepted risk. ## Why features are not folders @@ -65,7 +76,8 @@ A feature is a capability that can be understood and assessed on its own. This distinction matters because the graph is meant to support a release decision. A list of folders says little about user impact. A list of capabilities -with their expected behaviors, proof, and gaps shows where the risk is. +with their expected behaviors, mapped methods, evidence, and gaps shows where +the risk is. A useful test is: **if this failed, can I explain what would be broken for a user or operator?** If not, it may not be a feature. @@ -87,8 +99,9 @@ views. A view selects features; it does not copy or change them. ## What the graph is not The graph is not a test report. A report says what ran; the graph says what -matters and how the available proof relates to it. +matters and how the available evidence relates to it. The graph is also not a score. The [four scores](the-four-scores.md) are readouts -calculated from the graph and, for runtime quality, from observations. You improve -the underlying checks, proof, software, or review—not the number directly. +calculated from the graph and, for runtime quality, from observations. You +improve the underlying intent, checks, verification methods, software, evidence, +or validation—not the number directly. diff --git a/docs/concepts/terminology.md b/docs/concepts/terminology.md new file mode 100644 index 0000000..d1979ed --- /dev/null +++ b/docs/concepts/terminology.md @@ -0,0 +1,137 @@ +# Correctness terminology + +Shiplight Quality uses validation, verification, evidence, and observation for +different relationships. Keeping them separate matters as Quality expands from +empirical testing toward reasoning methods such as static analysis and formal +verification. + +## The model + +```text +product intent / PRD + │ + │ intent validation + ▼ +specification and quality claims + │ + │ verification + ▼ +┌──────────────────────────────┬──────────────────────────────┐ +│ reasoning-based verification │ empirical verification │ +│ review and static analysis │ tests and runtime checks │ +│ argument or formal proof │ execution and observation │ +└──────────────────────────────┴──────────────────────────────┘ + │ + ▼ +evidence → confidence → human decision +``` + +## Validation + +**Validation** asks whether the specification and quality claims accurately +represent product intent: are these the right requirements and expected +behaviors? + +In Quality, a person validates the proposed feature boundaries, priorities, and +complete list of checks against the PRD, specification, and other accepted +product sources. The current file format records parts of this decision through +feature status, priority provenance, and `checks_reviewed`. + +Validation does not establish that the implementation satisfies a claim. It +establishes that the claim is one the product actually intends to make. + +Quality also has a `validate` command that checks a YAML document against the +quality-map contract. Call this **schema validation** when the distinction +matters. Schema validation does not validate product intent. + +## Verification + +**Verification** asks whether the implementation or observed behavior satisfies +a quality claim. It includes both reasoning-based verification and empirical +verification. + +A **verification method** is a repeatable or inspectable way to evaluate that +relationship. Verification can follow two broad paths: + +- **Reasoning-based verification** examines the implementation or a model of it. + Examples include review, static analysis, model checking, and formal + verification. +- **Empirical verification** executes or monitors the system, records an + observation, and compares it with the claim. Examples include unit, + integration, and end-to-end tests, production telemetry, and manual runtime + procedures. + +Formal verification is therefore one specific reasoning-based verification +method. The unqualified word *verification* includes both reasoning and +empirical methods. + +## Observation + +An **empirical observation** is a recorded fact about executed or production +behavior. Empirical verification compares that observation with a quality claim. + +A **runtime observation** is also the name of Quality's standardized result +record. It says that an executable verification method passed, failed, errored, +or was skipped at a particular revision and time. This transport record can +carry the outcome of a reasoning tool such as a static checker. Recording that +outcome does not make the underlying method empirical; the method's evidence +still comes from reasoning about the implementation or its model. + +An observation does not support a quality claim merely because it exists. It +must resolve to the relevant verification method and be evaluated against the +expected outcome. Quality keeps the method-to-observation join explicit so a +missing result is not mistaken for a passing one. + +## Evidence + +**Evidence** is information that supports or challenges a quality claim. It can +come from reasoning or from evaluated observations. Its strength depends on the +method, scope, context, provenance, independence, and current result. + +The checked-in quality graph maps each claim to the verification methods and +artifacts expected to provide evidence. Runtime observations then record what +happened when executable methods were evaluated. Evidence confidence describes +the strength of that setup; runtime quality describes the observed outcomes. + +## Proof + +**Proof** is reserved for a deductive argument that establishes a proposition +under stated assumptions, including a machine-checked formal proof. Tests, +telemetry, and ordinary runtime observations provide empirical evidence; they +are not called proofs in user-facing Quality terminology. + +Some current machine-readable names predate this terminology. In particular, +`proof_gap` is the existing quality-map field for an evidence gap. Documentation +may show that literal name when explaining the file format, but prose should use +**evidence gap**. Compatibility names do not define the conceptual vocabulary. + +Formal proof is not yet a first-class method type in the current quality-map +contract. Treating one as `static` or `other` would not capture its theorem, +assumptions, proof artifact, or checker. Future formal-verification support +should extend the contract without changing the distinctions on this page. + +## Human decisions + +Human validation and human approval are related but distinct: + +- A person **validates intent** by confirming that features and checks represent + what the product is meant to do. +- A person **approves a decision** when accepting risk, setting policy, or making + a release decision. + +Quality records these decisions but does not make them. Avoid using +*ratification* as a general synonym for validation; it describes a formal act of +adoption, not the work of comparing claims with product intent. + +## Preferred wording + +| Avoid | Prefer | +| --- | --- | +| proof definition | verification method or mapped verification method | +| proof artifact | verification artifact or evidence artifact | +| proof type | verification method type or evidence type | +| proof policy / proof requirement | verification policy / verification requirement | +| proof gap | evidence gap (use `proof_gap` only for the literal field) | +| runtime proof | runtime evidence or observed result | +| proof producer | evidence producer | +| ratify a feature or check list | validate the feature or check list | diff --git a/docs/concepts/the-four-scores.md b/docs/concepts/the-four-scores.md index 7b589d8..ac9f56c 100644 --- a/docs/concepts/the-four-scores.md +++ b/docs/concepts/the-four-scores.md @@ -9,18 +9,18 @@ information to decide whether this release is ready?** | Score | Question | What it reads | | --- | --- | --- | -| **Quality** | Are the expected behaviors holding right now? | Latest test and proof results | -| **Coverage** | Does every expected behavior have connected proof? | Quality maps saved with the project | -| **Evidence confidence** | How convincing is that proof? | Proof type, runtime context, and proof requirements | -| **Structure confidence** | Is the list of expected behaviors trustworthy? | Their origin and whether a person reviewed them | +| **Quality** | Are the expected behaviors holding right now? | Latest observed verification results | +| **Coverage** | Does every expected behavior have a mapped verification method? | Quality maps saved with the project | +| **Evidence confidence** | How convincing are the mapped methods? | Method type, runtime context, and verification requirements | +| **Structure confidence** | Is the list of expected behaviors trustworthy? | Their origin and whether a person validated them | Priority weights affect every score. Quality does not choose your release threshold; your team does. ## Quality -Quality is the runtime score. It evaluates what happened when each expected -behavior's proof last ran in the selected assessment: +Quality is the runtime score. It evaluates the latest observation resolved to +each expected behavior's mapped verification method in the selected assessment: - passing: full credit; - mixed results: partial credit; @@ -33,35 +33,36 @@ results: a project with no observation set, or one whose results could not be loaded, still reports Coverage, Evidence confidence, and Structure confidence. Quality is most directly tied to release decisions, but a high score based on -incomplete, weak, or unreviewed checks is not a strong release signal. +incomplete, weak, or unvalidated checks is not a strong release signal. ## Coverage Coverage reads the saved quality maps and checks whether every expected behavior -has proof. Manual, indirect, incomplete, or policy-violating proof earns partial -credit. Running tests does not change coverage; mapping meaningful proof, -connecting existing proof, or resolving a proof gap does. +has a verification method. Manual, indirect, incomplete, or policy-violating +methods earn partial credit. Running tests does not change coverage; mapping a +meaningful method, connecting an existing method, or resolving an evidence gap +does. -Raising a proof requirement can lower Coverage, Evidence confidence, and the -static quality readout, but it does not change runtime results or Structure -confidence. +Raising a verification requirement can lower Coverage, Evidence confidence, +and the static quality readout, but it does not change runtime results or +Structure confidence. A feature without a quality map lowers coverage because none of its expected -behaviors can be evaluated. An expected behavior without proof becomes +behaviors can be evaluated. An expected behavior without a mapped method becomes unobserved at runtime, but coverage is not multiplied into Quality. ## Evidence confidence -Evidence confidence measures the strength of the proof setup, not whether the -latest run passed. Automated proof is stronger than a supporting or manual -artifact; proof is stronger when it runs in CI or when multiple automated -methods check the same expected behavior. A proof requirement can also require -a particular type or runtime context. +Evidence confidence measures the strength of the verification setup, not +whether the latest run passed. An automated method is stronger than a supporting +or manual artifact; evidence is stronger when a method runs in CI or when +multiple automated methods check the same expected behavior. A verification +requirement can also require a particular type or runtime context. The engine evaluates those recorded facts. It does not know that a unit test is too narrow for an important behavior unless the map records that requirement. Therefore a failing end-to-end test can still have high evidence confidence: -the proof is trustworthy and is reporting a problem. +the method is trustworthy and its observation is reporting a problem. ## Structure confidence @@ -70,18 +71,19 @@ and whether their origins are trustworthy. A behavior from a specification or person receives more confidence than one generated by an agent, inferred from code, or missing a source. -When a feature is confirmed and a person approves its complete list of checks, -those checks receive full structure confidence. Original sources remain visible; -approval does not turn an agent-drafted list into human authorship. Running tests -does not change this score; honest provenance and human review do. +When a feature is confirmed and a person validates its complete list of checks, +those checks receive full structure confidence. Original sources remain +visible; validation does not turn an agent-drafted list into human authorship. +Running tests does not change this score; honest provenance and human validation +do. ## Read the four scores together | What you see | What it means | | --- | --- | | High quality, strong coverage and confidence | Measured behaviors are holding and the result has a strong foundation. Apply your release threshold. | -| High quality, low coverage | Measured behaviors are holding, but important areas may have no proof. | -| High quality, low evidence confidence | Available proof passes, but may not be strong enough to rely on. | +| High quality, low coverage | Measured behaviors are holding, but important areas may have no verification method. | +| High quality, low evidence confidence | Available results pass, but their methods may not be strong enough to rely on. | | High quality, low structure confidence | Something passes, but the team has not established enough trust in what is being measured. | Quality uses runtime results and priority. The other three scores are not @@ -90,11 +92,11 @@ ingredients in its formula; they tell you how much confidence to place in it. ## Read them honestly A high number is not the goal. Scores can rise without safer software by -publishing only successful runs, removing expected behaviors without proof, -describing weak proof as strong, or recording provenance or human approval that -did not happen. If the score rises without better software, proof, or review, -investigate why. **Unavailable** means there is not enough information to -calculate the score; it is not zero, pass, or failure. +publishing only successful runs, removing expected behaviors without evidence, +describing weak evidence as strong, or recording provenance or human validation +that did not happen. If the score rises without better software, evidence, or +validation, investigate why. **Unavailable** means there is not enough +information to calculate the score; it is not zero, pass, or failure. ## How to see the scores diff --git a/docs/concepts/trust-boundaries.md b/docs/concepts/trust-boundaries.md index 50ee7e3..278bbb7 100644 --- a/docs/concepts/trust-boundaries.md +++ b/docs/concepts/trust-boundaries.md @@ -1,17 +1,17 @@ # How Quality earns your trust -Quality is trustworthy only when the system evaluating proof cannot quietly -change the proof, the score, or a decision that belongs to you. +Quality is trustworthy only when the system evaluating evidence cannot quietly +change the evidence, the score, or a decision that belongs to you. ## The guarantees -### Proof remains independent +### Evidence remains independent Systems that build and test software produce facts and artifacts. Quality reads and evaluates them; it does not create the tests it assesses or change reported results. A result source may retrieve an existing observation file, but cannot -reinterpret it or decide whether proof passed. New runner formats must be -normalized before evaluation. +reinterpret it or decide whether a verification method passed. New runner +formats must be normalized before evaluation. ### Scores are deterministic @@ -40,9 +40,9 @@ for review. ### Human decisions stay human Agents may propose features, checks, priorities, or accepted risks. They cannot -approve their own proposals, mark a person's review complete, or accept risk on -that person's behalf; passing tests are not human approval. Structure confidence -reaches full trust only when a person confirms the feature and reviews its +validate their own proposals, mark a person's validation complete, or accept +risk on that person's behalf; passing tests are not human validation. Structure +confidence reaches full trust only when a person validates the feature and its complete set of checks. ## Implementation diff --git a/docs/concepts/who-decides-what.md b/docs/concepts/who-decides-what.md index 5b217da..008128a 100644 --- a/docs/concepts/who-decides-what.md +++ b/docs/concepts/who-decides-what.md @@ -1,7 +1,7 @@ # Who decides what An agent can investigate, prepare, and recommend. People remain responsible -for product intent, priority, approval, and accepted risk. +for product intent, priority, validation, and accepted risk. ## What an agent can prepare @@ -9,9 +9,9 @@ An agent may: - propose features and priorities; - draft checks in product language; -- connect existing proof and report gaps; +- connect existing verification methods and report evidence gaps; - configure authorized, mechanical result collection; and -- suggest a stronger proof requirement or a possible accepted risk. +- suggest a stronger verification requirement or a possible accepted risk. These are proposals or implementation tasks, not human decisions. @@ -22,12 +22,12 @@ These are proposals or implementation tasks, not human decisions. | Feature boundary | This is a product capability, not a folder or implementation detail. | | Priority | The importance reflects what the team would protect or block a release for. | | Origin | The recorded source of each check is truthful. | -| Check-list approval | The complete list of expected behaviors is correct and complete. | -| Proof requirement | A check needs a particular method, context, or delivery gate. | +| Intent validation | The feature and complete list of expected behaviors accurately represent product intent. | +| Verification requirement | A check needs a particular method, context, or delivery gate. | | Accepted risk | The team understands and chooses to tolerate the gap. | An agent may record a decision after you state it explicitly. It must not infer -approval from silence, passing tests, or its own confidence. +validation from silence, passing tests, or its own confidence. ## How structure confidence changes @@ -35,24 +35,24 @@ Structure confidence uses two inputs: 1. **Origin:** Checks from a specification or person receive more confidence than agent-generated checks, code inferences, or checks with no source. -2. **Review:** When a feature is confirmed and a person approves its complete +2. **Intent validation:** When a person validates a feature and its complete list of checks, all checks in that feature receive full structure confidence. -The original source remains visible after approval. Feature priority affects +The original source remains visible after validation. Feature priority affects aggregate weighting; recording `priority_provenance: human` protects a human-set priority but does not add separate structure-confidence points. -## Raising or lowering the proof bar +## Raising or lowering the verification bar -- **Require stronger proof:** A check may be required to use a particular method +- **Require stronger evidence:** A check may be required to use a particular method or delivery gate. Until met, structural coverage and evidence confidence may fall; existing runtime results do not change. - **Accept a known gap:** A person may tolerate a specific gap category. The gap remains visible but stops counting as open. Only `missing`, `manual-only`, and `weak` remove a structural scoring penalty when accepted. -An agent may make a proof requirement stricter, but may not weaken it or accept -risk on your behalf. +An agent may make a verification requirement stricter, but may not weaken it or +accept risk on your behalf. ## Questions for review @@ -62,5 +62,5 @@ risk on your behalf. - Does each priority match the consequence of failure? - Is any accepted risk genuinely understood and tolerable? -See [Review and approve the graph](../how-to/review-and-ratify.md) for the -practical workflow. +See [Validate the graph](../how-to/validate-the-graph.md) for the practical +workflow. diff --git a/docs/how-to/accept-a-known-gap.md b/docs/how-to/accept-a-known-gap.md index e4e29ce..3b87831 100644 --- a/docs/how-to/accept-a-known-gap.md +++ b/docs/how-to/accept-a-known-gap.md @@ -6,15 +6,15 @@ living with it is an explicit decision—for example, relying on a manual release check for a low-use internal screen. -Acceptance records risk; it does not add proof or change a test result. +Acceptance records risk; it does not add evidence or change a test result. ## Ask your agent Name the check and the gap category you are accepting: ```text -We accept the missing end-to-end proof on the admin export check as tolerated -risk. Add the "missing" category to that check's accepted gaps. +We accept the missing end-to-end verification on the admin export check as +tolerated risk. Add the "missing" category to that check's accepted gaps. ``` To reverse the decision: @@ -27,8 +27,8 @@ category from its accepted gaps. ## What the agent does The agent adds or removes one category in the check's `accepted_gaps` list. The -underlying status and proof remain unchanged, so readers can still see what is -missing. +underlying status and mapped verification methods remain unchanged, so readers +can still see what is missing. The current file format stores the accepted category but does **not** have a dedicated field for the reason, owner, or expiry date. Record that context in @@ -77,8 +77,8 @@ Before accepting, ask: [Act on a low score](act-on-a-weak-score.md). **Accepted risks are accumulating.** Review whether the list of checks is wrong, -whether proof investment is being postponed indefinitely, or whether accepted -decisions need an expiry process outside Quality. +whether verification investment is being postponed indefinitely, or whether +accepted decisions need an expiry process outside Quality. **The score did not change.** That is expected for state categories such as `stale` or `failing`. Acceptance changes open-gap reporting, not the observed diff --git a/docs/how-to/act-on-a-weak-score.md b/docs/how-to/act-on-a-weak-score.md index ed733c9..4b74877 100644 --- a/docs/how-to/act-on-a-weak-score.md +++ b/docs/how-to/act-on-a-weak-score.md @@ -23,29 +23,30 @@ Why is evidence confidence low for the payments feature? | Score | What a low value can mean | What usually helps | | --- | --- | --- | -| **Quality** | Checks failed, errored, were skipped, have mixed results, or have no matching observation | Fix the behavior or proof pipeline, then rerun it | -| **Coverage** | Proof is missing, manual, indirect, incomplete, or below a declared policy | Connect or add the appropriate proof and resolve the recorded gap | -| **Evidence confidence** | Proof is absent, non-automated, single-layer, ungated, or constrained by an unmet policy | Use an appropriate automated method, meaningful second method, or delivery gate | -| **Structure confidence** | Check origin is weak or undeclared, or the complete list has not been approved for a confirmed feature | Correct the recorded origin and ask a person to review the list | +| **Quality** | Checks failed, errored, were skipped, have mixed results, or have no matching observation | Fix the behavior or result pipeline, then rerun it | +| **Coverage** | A verification method is missing, manual, indirect, incomplete, or below a declared policy | Connect or add the appropriate method and resolve the recorded evidence gap | +| **Evidence confidence** | Methods are absent, non-automated, single-layer, ungated, or constrained by an unmet policy | Use an appropriate automated method, meaningful second method, or delivery gate | +| **Structure confidence** | Check origin is weak or undeclared, or the complete list has not been validated for a confirmed feature | Correct the recorded origin and ask a person to validate the list | Do not average these explanations. A passing runtime result does not repair a -coverage gap, and stronger proof does not approve the product's expected behaviors. +coverage gap, and stronger evidence does not validate the product's expected +behaviors. -## Separate missing proof from missing results +## Separate missing methods from missing observations Two problems can both produce an unobserved runtime check: -- **No proof is mapped.** Coverage is also weak. The fix is to identify or create - appropriate proof and connect it to the check. -- **Proof is mapped but no result matched it.** Coverage may look healthy, while - runtime quality is weak. The fix is the result pipeline or the path/test-case - join—not another test. +- **No verification method is mapped.** Coverage is also weak. The fix is to + identify or create an appropriate method and connect it to the check. +- **A method is mapped but no observation matched it.** Coverage may look + healthy, while runtime quality is weak. The fix is the result pipeline or the + path/test-case join—not another test. -Ask for the resolution audit before creating new proof: +Ask for the resolution audit before creating a new method: ```text -For each unobserved check, tell me whether proof is missing or whether a result -failed to match. Show the resolution audit. +For each unobserved check, tell me whether a verification method is missing or +whether an observation failed to match. Show the resolution audit. ``` See [Make CI results count](make-ci-results-count.md) when results are not @@ -54,12 +55,12 @@ reaching the graph. ## Decisions that remain yours - Whether an expected behavior belongs in the graph at all. -- Whether a proof policy reflects the risk correctly. -- Whether the list of checks is complete and approved. +- Whether a verification policy reflects the risk correctly. +- Whether the list of checks is complete and validated against product intent. - Whether a known gap is acceptable. -Removing a real expected behavior or narrowing a release view solely to raise a score -hides risk rather than resolving it. +Removing a real expected behavior or narrowing a release view solely to raise a +score hides risk rather than resolving it. ## Verify the improvement @@ -72,14 +73,14 @@ hides risk rather than resolving it. ## Troubleshooting **The number rose but the underlying condition did not improve.** Check whether -scope was removed, proof was mislabeled, a policy was weakened, or risk was -accepted only to clear a warning. Restore the honest input and treat unexpected -score movement as a defect. +scope was removed, a verification method was mislabeled, a policy was weakened, +or risk was accepted only to clear a warning. Restore the honest input and treat +unexpected score movement as a defect. **More tests did not help.** Confirm that their paths and optional test-case names match the quality map and that the selected observation set actually loads their results. **Structure confidence remains low.** Recording a truthful origin can help, but -an agent cannot provide human approval. Review the feature and its complete -list of checks yourself. +an agent cannot provide human validation. Compare the feature and its complete +list of checks with the accepted product intent yourself. diff --git a/docs/how-to/inspect-in-the-browser.md b/docs/how-to/inspect-in-the-browser.md index 49d6d23..235f64c 100644 --- a/docs/how-to/inspect-in-the-browser.md +++ b/docs/how-to/inspect-in-the-browser.md @@ -1,7 +1,7 @@ # Use Quality Explorer -**Outcome:** A local web view of the project's features, checks, proof, results, -and gaps. +**Outcome:** A local web view of the project's features, checks, verification +methods, evidence, results, and gaps. **Use this when:** You want to understand the project without reading YAML, walk through risk in a review meeting, or inspect why a result did not connect to a @@ -29,7 +29,7 @@ graph and all four score workflows require the `.quality/` setup described in Quality Explorer can: -- Scan the selected project and show its feature and proof graph. +- Scan the selected project and show its feature and evidence graph. - Display available structural scores and runtime quality. - Show checks, gaps, source files, and evidence relationships. - Load configured observation sources or sets. @@ -52,7 +52,7 @@ still make an outbound request and may require a token; “read-only” does not ## Decisions that remain yours -The interface can show an unconfirmed feature, unreviewed list of checks, proposed +The interface can show an unvalidated feature or list of checks, proposed policy, or accepted-risk option. It cannot make those decisions for you. See [Who decides what](../concepts/who-decides-what.md). diff --git a/docs/how-to/make-ci-results-count.md b/docs/how-to/make-ci-results-count.md index c721359..8f45c8c 100644 --- a/docs/how-to/make-ci-results-count.md +++ b/docs/how-to/make-ci-results-count.md @@ -26,7 +26,7 @@ Without that permission, the agent should propose the workflow change and stop. ## What the agent does -1. Finds the job that already runs the relevant proof. +1. Finds the job that already runs the relevant verification method. 2. Converts JUnit or Playwright output—or records an existing gate outcome—into the standard `quality-observations.json` format. 3. Includes the commit and run metadata. @@ -51,13 +51,14 @@ Check all three stages; success at one stage does not guarantee the next. Confirm that `quality-observations.json` is present on both successful and failed runs. Publishing only green runs creates a biased quality score. -### 2. Results match mapped proof +### 2. Results match mapped verification methods Ask: ```text -Assess the project. Report how many observations matched proof, how many were -unmatched, and how many were ambiguous. Show the resolution audit. +Assess the project. Report how many observations matched mapped verification +methods, how many were unmatched, and how many were ambiguous. Show the +resolution audit. ``` Matching uses the evidence path and optional test-case name. Results arriving in diff --git a/docs/how-to/map-a-feature.md b/docs/how-to/map-a-feature.md index 4a50624..67310d8 100644 --- a/docs/how-to/map-a-feature.md +++ b/docs/how-to/map-a-feature.md @@ -1,7 +1,8 @@ # Map a feature **Outcome:** One capability gets a readable list of the expected behaviors it -must keep, the existing proof for each behavior, and the gaps that remain. +must keep, the mapped verification methods for each behavior, and the evidence +gaps that remain. **Use this when:** After initial setup, when adding the next important feature, or when a feature's responsibilities have changed. @@ -21,7 +22,7 @@ Or describe the feature: ```text Map the feature responsible for recording uploads. Show what must remain true, -what proof already exists, and what is still missing. +which verification methods already exist, and what evidence is still missing. ``` ## What the agent does @@ -29,17 +30,18 @@ what proof already exists, and what is still missing. - Reads accepted specifications when available, then relevant code, tests, and CI configuration. - Drafts checks in product language. -- Connects existing tests, workflows, and other proof. +- Connects existing tests, analyses, workflows, and other verification methods. - Records the source of the checks honestly. -- Records missing or inadequate proof instead of omitting the expected behavior. +- Records missing or inadequate evidence instead of omitting the expected + behavior. -The agent does not write tests during feature mapping. Creating missing proof is -a separate implementation task. +The agent does not write tests during feature mapping. Creating a missing +verification method is a separate implementation task. ## Decisions that remain yours **Whether the checks are right.** The first list is a proposal. Review it using -[Review and approve the graph](review-and-ratify.md). +[Validate the graph](validate-the-graph.md). **How important each check is.** When accepted source material provides a priority, the map can use it. Otherwise the check should remain `UNKNOWN` until @@ -62,24 +64,25 @@ Also check: - Are important failure modes represented? - Does the list include lessons from past incidents or contractual commitments? -- Is every proof path real and repository-relative? +- Is every verification artifact path real and repository-relative? - Where a check names a specific test case, does that name match what the test reporter emits? - Are gaps visible rather than hidden by a short list? -Checks participate in scoring before human approval; approval changes how much -structure confidence the list receives. Do not approve merely to make the graph -“start counting.” +Checks participate in scoring before human validation; validation changes how +much structure confidence the list receives. Do not mark a list reviewed merely +to make the graph “start counting.” ## How runtime results connect -Quality joins a result to proof primarily through its file path and, when +Quality joins an observation to a mapped verification method primarily through +its file path and, when present, an optional test-case name. Test-case matching ignores case and surrounding whitespace; file paths may match as repository-relative suffixes. -Renaming a proof file can produce a missing-file scan warning. Renaming a pinned -test case can leave observations unmatched. Use the resolution audit when a -previously observed check becomes unobserved. +Renaming a verification artifact can produce a missing-file scan warning. +Renaming a pinned test case can leave observations unmatched. Use the resolution +audit when a previously observed check becomes unobserved. ## Troubleshooting @@ -95,5 +98,5 @@ instead of accepted intent. Explain the user or operational outcome, or ## Next steps -- [Review and approve the list of checks](review-and-ratify.md). +- [Validate the list of checks](validate-the-graph.md). - [Act on the highest-value gap](act-on-a-weak-score.md). diff --git a/docs/how-to/require-stronger-proof.md b/docs/how-to/require-stronger-proof.md deleted file mode 100644 index df072f9..0000000 --- a/docs/how-to/require-stronger-proof.md +++ /dev/null @@ -1,84 +0,0 @@ -# Require stronger proof - -**Outcome:** One check is treated as incomplete until its proof meets a standard -you specify. - -**Use this when:** An expected behavior is more important than its current proof justifies, -especially for money, permissions, privacy, data integrity, or failures that -would be expensive and quiet. - -## Ask your agent - -Require a particular proof method: - -```text -For the check "A customer is never charged twice for one order," require an -end-to-end test before the structural assessment treats it as covered. -``` - -Require a runtime context: - -```text -That check must also run in our release pipeline, not only on a developer's -machine. -``` - -## What the agent does - -The agent records a proof policy on that check. The policy can require: - -- A delivery gate such as CI or release. -- One or more proof types. -- One or more named runtime contexts. - -Until the mapped proof meets the requirement, the check remains structurally -partial and the missing requirement appears in recommendations. - -## Decision that remains yours - -You own the proof standard and any later decision to weaken or remove it. An -agent may propose or add a stricter requirement because that cannot manufacture -confidence. It must not lower a requirement you set unless you explicitly direct -it to do so. - -## How scores respond - -If the check already met the requirement, no score changes. Otherwise: - -- Coverage can fall because the structural proof is now partial. -- Evidence confidence can fall from high to medium. -- The static quality readout in generated analysis can fall. -- Runtime quality does not change. The recorded test result still says what - happened in that run. -- Structure confidence does not change. - -A lower structural score after raising the bar is not a regression in the -software. It is a more demanding and more honest description of the proof. - -## Verify the result - -- The named check contains the intended proof requirement. -- The assessment identifies exactly which requirement is unmet. -- No unrelated checks changed. -- The requirement is satisfied only by evidence with the required type or - context. - -You can ask: - -```text -Does this check meet the proof requirement now? Show which evidence satisfies -each part. -``` - -## Troubleshooting - -**Nothing changed.** The existing proof may already satisfy the new policy. Ask -the agent to show the matching evidence and context. - -**You intended a team-wide standard.** Proof policies are attached to individual -checks. If the same rule belongs everywhere, document a team testing standard -and decide how it should be represented rather than repeating an exception on -every check. - -**Runtime quality stayed the same.** That is expected. A proof policy changes the -structural assessment; it does not rewrite an observation that already happened. diff --git a/docs/how-to/set-up-quality.md b/docs/how-to/set-up-quality.md index eff5e98..00d5da4 100644 --- a/docs/how-to/set-up-quality.md +++ b/docs/how-to/set-up-quality.md @@ -1,7 +1,7 @@ # Set up Quality **Outcome:** A first, reviewable map of your project, its features, the expected -behaviors one feature must keep, and the proof already available. +behaviors one feature must keep, and the verification methods already available. **Use this when:** You are introducing Quality to a repository. It works with a detailed specification, with existing code and tests, or with a partially built @@ -27,14 +27,14 @@ The agent will: existing `.quality/` files. 2. Explain what parts of the quality graph already exist and what is missing. 3. Propose a small set of product capabilities as features. -4. Map one high-priority feature first, including its checks, existing proof, - and honest gaps. +4. Map one high-priority feature first, including its checks, existing + verification methods, and honest evidence gaps. 5. Report the structural scores and, if runtime results are already connected, the runtime quality score. -The agent maps existing proof. It does not create tests or change how a test -decides pass or fail. If connecting existing CI results requires a workflow -edit, it must ask for explicit permission first. +The agent maps existing verification methods. It does not create tests or change +how a test decides pass or fail. If connecting existing CI results requires a +workflow edit, it must ask for explicit permission first. ## Decisions that remain yours @@ -43,12 +43,12 @@ The first map is a proposal. You decide: - Whether the proposed features are real product capabilities. - Which features and checks matter most. - Whether the recorded origin of each check is accurate. -- Whether the complete list of checks is correct and ready for approval. +- Whether the complete list of checks accurately represents product intent. - Whether any known gap is acceptable. An agent-generated or code-inferred map normally begins with less than full structure confidence. That is useful information, not a setup failure. See -[Review and approve the graph](review-and-ratify.md). +[Validate the graph](validate-the-graph.md). ## Verify the result @@ -56,8 +56,8 @@ structure confidence. That is useful information, not a setup failure. See capabilities rather than folders or packages. - At least one feature has a quality map under `.quality/evidence/`. - Its checks make sense without requiring the reader to know function names. -- Existing proof is connected where possible and missing proof is shown as a - gap. +- Existing verification methods are connected where possible and missing + evidence is shown as a gap. - Any unavailable score includes an explanation. The runtime quality score is often unavailable on the first pass because no @@ -87,6 +87,6 @@ Continue with [Make CI results count](make-ci-results-count.md). ## Next steps -1. [Review and approve the first feature](review-and-ratify.md). +1. [Validate the first feature](validate-the-graph.md). 2. [Connect CI results](make-ci-results-count.md). 3. [Map the next important feature](map-a-feature.md). diff --git a/docs/how-to/set-verification-requirements.md b/docs/how-to/set-verification-requirements.md new file mode 100644 index 0000000..c0478c3 --- /dev/null +++ b/docs/how-to/set-verification-requirements.md @@ -0,0 +1,91 @@ +# Set verification requirements + +**Outcome:** One check is treated as incomplete until its verification methods +meet a standard you specify. + +**Use this when:** An expected behavior is more important than its current +evidence justifies, especially for money, permissions, privacy, data integrity, +or failures that would be expensive and quiet. + +## Ask your agent + +Require a particular verification method: + +```text +For the check "A customer is never charged twice for one order," require an +end-to-end test before the structural assessment treats it as covered. +``` + +Require a runtime context: + +```text +That check must also run in our release pipeline, not only on a developer's +machine. +``` + +## What the agent does + +The agent records a verification policy on that check. The policy can require: + +- A delivery gate such as CI or release. +- One or more verification method types. +- One or more named runtime contexts. + +Until the mapped methods meet the requirement, the check remains structurally +partial and the missing requirement appears in recommendations. + +## Decision that remains yours + +You own the verification standard and any later decision to weaken or remove +it. An agent may propose or add a stricter requirement because that cannot +manufacture confidence. It must not lower a requirement you set unless you +explicitly direct it to do so. + +## How scores respond + +If the check already met the requirement, no score changes. Otherwise: + +- Coverage can fall because the mapped verification setup is now partial. +- Evidence confidence can fall from high to medium. +- The static quality readout in generated analysis can fall. +- Runtime quality does not change. The recorded test result still says what + happened in that run. +- Structure confidence does not change. + +A lower structural score after raising the bar is not a regression in the +software. It is a more demanding and more honest description of the available +evidence. + +## Verify the result + +- The named check contains the intended verification requirement. +- The assessment identifies exactly which requirement is unmet. +- No unrelated checks changed. +- The requirement is satisfied only by evidence with the required type or + context. + +You can ask: + +```text +Does this check meet the verification requirement now? Show which evidence +satisfies each part. +``` + +## Troubleshooting + +**Nothing changed.** The existing methods may already satisfy the new policy. Ask +the agent to show the matching evidence and context. + +**You intended a team-wide standard.** Verification policies are attached to +individual checks. If the same rule belongs everywhere, document a team +verification standard and decide how it should be represented rather than +repeating an exception on every check. + +**Runtime quality stayed the same.** That is expected. A verification policy +changes the structural assessment; it does not rewrite an observation that +already happened. + +## Next steps + +- [Validate the graph](validate-the-graph.md) +- [Act on a weak score](act-on-a-weak-score.md) diff --git a/docs/how-to/review-and-ratify.md b/docs/how-to/validate-the-graph.md similarity index 61% rename from docs/how-to/review-and-ratify.md rename to docs/how-to/validate-the-graph.md index e50ea93..edf5eb7 100644 --- a/docs/how-to/review-and-ratify.md +++ b/docs/how-to/validate-the-graph.md @@ -1,7 +1,7 @@ -# Review and approve the graph +# Validate the graph -**Outcome:** Agent proposals become explicit human decisions, with their origins -and approvals recorded separately. +**Outcome:** A person confirms that proposed features and checks accurately +represent product intent, with their origins and validation recorded separately. **Use this when:** A project or feature has been mapped and is waiting for your review. @@ -15,11 +15,13 @@ Show me the Quality features, priorities, and lists of checks that still need a person's decision. Put the highest-priority items first. ``` -Then respond in ordinary language. For example: +Then compare the proposal with the PRD, specification, and other accepted +product sources. Respond in ordinary language. For example: ```text -Feature 001 is a real capability and its priority is P0. I have read and -approve the complete list of checks for 001. +Feature 001 is a real capability and its priority is P0. I compared the complete +list of checks for 001 with the billing PRD. It accurately represents the +required behavior and nothing important is missing. Mark it reviewed. ``` Disagreement is equally useful: @@ -27,18 +29,18 @@ Disagreement is equally useful: ```text Feature 002 is a folder, not a product capability. Merge it into 001. The third check on 001 describes a test rather than a customer-facing expected behavior; -rewrite it as an outcome and ask for review. +rewrite it as an outcome and ask me to validate the revised list. ``` ## What the agent does -The agent shows the proposed decisions and records only the ones you state. It -keeps two facts separate: +The agent shows the proposed structure and records only the validations you +state. It keeps two facts separate: - Where a feature or check originally came from. -- Whether a person later reviewed and approved it. +- Whether a person later validated it against product intent. -An agent-drafted check remains recorded as agent-drafted after approval. That +An agent-drafted check remains recorded as agent-drafted after validation. That history is useful and must not be rewritten as human authorship. ## Decisions that remain yours @@ -48,7 +50,7 @@ history is useful and must not be rewritten as human authorship. | Confirm the feature | Is this a genuine capability of the product? | | Set the priority | How serious would failure be for users or a release? | | Confirm the origin | Did these checks come from a specification, a person, an agent, or an inference from code? | -| Approve the checks | Are these the right expected behaviors, and is anything important missing? | +| Validate the checks | Do these expected behaviors accurately and completely represent product intent? | ### What to look for @@ -62,14 +64,14 @@ the behavior the product must preserve. fragile behavior the repository could not reveal. - **Unhelpful priorities:** A list in which everything has the same priority does not identify what matters most. -- **An inaccurate origin:** Human approval does not turn an agent-generated +- **An inaccurate origin:** Human validation does not turn an agent-generated check into a human-authored check. -## How approval affects scores +## How validation affects scores The recorded origin contributes to structure confidence immediately. Separately, -approving the complete list of checks gives every check full structure confidence -when the feature is also confirmed. +validating the complete list of checks gives every check full structure +confidence when the feature is also confirmed. Priority affects how heavily checks count in all aggregate scores. The `priority_provenance` record protects a human-set priority from being overwritten, @@ -79,20 +81,21 @@ that field. ## Verify the result - The feature is no longer marked as a candidate if you confirmed it. -- The approved quality map records that the complete list of checks was reviewed. +- The validated quality map records that the complete list of checks was reviewed + (the `checks_reviewed` field). - The original check provenance is unchanged. - Structure confidence reaches full credit for that feature only when feature - confirmation and approval of the complete list are both present. -- Nothing you did not explicitly approve was marked approved. + confirmation and validation of the complete list are both present. +- Nothing you did not explicitly validate was marked reviewed. ## Troubleshooting -**Approval did not lift the checks to full structure confidence.** Confirm both +**Validation did not lift the checks to full structure confidence.** Confirm both requirements: the feature must not still be a candidate, and the complete list of checks must be marked reviewed. **The list is too long.** Review one high-priority feature at a time. A smaller -area with deliberate approval is more useful than a project-wide skim. +area with deliberate validation is more useful than a project-wide skim. **Most of the proposal is wrong.** Ask the agent to revise it. Finding that the model misunderstood the product is exactly what this review is designed to do. diff --git a/docs/verification/quality-prompt-sweep.md b/docs/verification/quality-prompt-sweep.md index 8be482e..97af016 100644 --- a/docs/verification/quality-prompt-sweep.md +++ b/docs/verification/quality-prompt-sweep.md @@ -73,19 +73,19 @@ language used the same Claude session; persisted-state checks used fresh ones. | Scoped bootstrap | **Pass with inconsistent provenance.** `/quality start, focus on the highest-risk user-facing workflow first` selected 001 with a defensible risk explanation and completed in about 11.5 minutes. Unlike the first bootstrap, it marked spec-backed features `candidate` and the map `agent_generated`; that conflicts with the spec-driven rules and the earlier run's `implemented`/`spec` result. | | Exact and natural feature mapping | **Pass.** Both `/quality map-feature 002-recorder-session-upload` and the natural-language recorder-upload example routed correctly, validated the map, and exposed real gaps. | | Help, status, and project mapping | **Pass.** `/quality help`, `/quality status`, and `/quality map-project` honored their read/write boundaries and reported persisted state. | -| Stronger proof policy | **Pass with an interpretation defect.** A real check accepted an e2e modality requirement, then a release-CI context requirement. The follow-up correctly found both unmet, but incorrectly said `require_gate` was also unmet: the engine treats existing `pr-ci` evidence as a gate, while `required_contexts: ["release-ci"]` expresses the narrower release requirement. | -| Accept and withdraw a gap | **Pass.** Explicit owner language added `missing` to `accepted_gaps`; the follow-up removed it without changing proof or test results. | -| Review and ratification | **Pass with response defects.** The review prompt prioritized pending human decisions. Explicit approval changed only `priority_provenance` and `checks_reviewed`, but the response said “three checks” and listed seven, and it added a reviewer email not supplied in the prompt. | +| Stronger verification policy | **Pass with an interpretation defect.** A real check accepted an e2e modality requirement, then a release-CI context requirement. The follow-up correctly found both unmet, but incorrectly said `require_gate` was also unmet: the engine treats existing `pr-ci` evidence as a gate, while `required_contexts: ["release-ci"]` expresses the narrower release requirement. | +| Accept and withdraw a gap | **Pass.** Explicit owner language added `missing` to `accepted_gaps`; the follow-up removed it without changing mapped methods or test results. | +| Intent validation | **Pass with response defects.** The review prompt prioritized pending human decisions. Explicit validation changed only `priority_provenance` and `checks_reviewed`, but the response said “three checks” and listed seven, and it added a reviewer email not supplied in the prompt. | | Diagnose weak scores | **Routes correctly, explanation needs correction.** Coverage diagnosis chose the unmapped shipped P1 feature as the smallest honest improvement. Evidence-confidence diagnosis mixed structure review into the evidence axis and used incorrect check totals. | -| Resolution audit without observations | **Pass with incorrect totals.** It correctly explained that no resolution audit exists before runtime acquisition and separated missing proof from join failure, but its category counts did not match its own lists. | +| Resolution audit without observations | **Pass with incorrect totals.** It correctly explained that no resolution audit exists before runtime acquisition and separated a missing verification method from join failure, but its category counts did not match its own lists. | | Saved view | **Pass.** The agent created a two-feature view with exact ids. Assessment then hit the known published-CLI requirement for an observation set; the static-assessment fix is tracked separately. | | CI observation connection | **Pass after real prerequisite repair.** The skill stopped at the producer boundary, required explicit CI authorization, preserved test semantics, emitted canonical observations, and created the smallest source/set. Local JUnit verification exposed broken graph join keys before configuration was written. | | Assessment before the new CI artifact exists | **Pass.** GitHub acquisition found the latest real run, reported the missing artifact, returned `0` matched/unmatched/ambiguous observations, preserved the three static scores, and left Quality unavailable. | | External product sources | **Pass.** Linear and Jira sources were recorded as human-selected but reported unavailable without inventing content. A GitHub URL resolving to the already-indexed local PRD was not duplicated. The follow-up accurately reported which sources were used and what changed. | -The disagreement example in `review-and-ratify.md` was not executed literally: +The disagreement example in `validate-the-graph.md` was not executed literally: it asserts that feature 002 is a folder and should merge into 001, which is -false for this repository. Its underlying correction/ratification path was +false for this repository. Its underlying correction/validation path was covered by the other human-decision scenarios. `spec-project` prompts and shell CLI examples are outside this Quality-prompt sweep. @@ -99,7 +99,7 @@ CLI examples are outside this Quality-prompt sweep. 2. **Generated evidence did not join real Vitest output.** Local JUnit conversion showed 57 of 57 pinned rows unmatched because the maps stored bare test labels while Vitest emits the full nested name. Three rows also cited the wrong test - file. Mechanical repairs raised a local runtime score from 31 to 77, proving + file. Mechanical repairs raised a local runtime score from 31 to 77, showing that the join keys—not the test results—caused most of the loss. 3. **Pinned and unpinned rows can become ambiguous.** After repairing the exact pins, 46 observations matched both a pinned row and an unpinned row for the @@ -135,13 +135,14 @@ CLI examples are outside this Quality-prompt sweep. tells readers to substitute an id from `.quality/project-map.yaml`. 5. **Session semantics should be stated in testing guidance.** Follow-up prompts should use an explicit `--resume `; independent documentation - prompts should run in a fresh session so they prove persisted files, not + prompts should run in a fresh session so they evaluate persisted files, not conversational memory. ## Verification boundary This sweep verified Quality prompt routing, scoped noninteractive permissions, -session persistence, bootstrap, mapping, human gates, proof policy, saved views, -CI wiring, runtime acquisition failure, resolution reporting, and external -source handling. It did not verify `spec-project`, development commands, or -external tracker retrieval because no Linear/Jira connector was available. +session persistence, bootstrap, mapping, human gates, verification policy, +saved views, CI wiring, runtime acquisition failure, resolution reporting, and +external source handling. It did not verify `spec-project`, development +commands, or external tracker retrieval because no Linear/Jira connector was +available. diff --git a/tests/agent/quality-evidence-type-consistency.md b/tests/agent/quality-evidence-type-consistency.md index 79a3b54..97eeb1d 100644 --- a/tests/agent/quality-evidence-type-consistency.md +++ b/tests/agent/quality-evidence-type-consistency.md @@ -2,7 +2,7 @@ ## Purpose -Verify that `/quality map-feature` classifies proof by its execution boundary +Verify that `/quality map-feature` classifies verification methods by their execution boundary and remains consistent with other feature maps without assuming every test in one file has the same type. @@ -26,4 +26,4 @@ The resulting proposal must: 3. Allow pinned cases to differ even though they share a path. 4. Detect and report the conflict for an identical path and case instead of silently preserving both classifications. -5. Leave tests, reports, and human-owned ratification fields unchanged. +5. Leave tests, reports, and human-owned validation fields unchanged. diff --git a/tests/agent/quality-runtime-join-identities.md b/tests/agent/quality-runtime-join-identities.md index 3a62af4..51696c5 100644 --- a/tests/agent/quality-runtime-join-identities.md +++ b/tests/agent/quality-runtime-join-identities.md @@ -28,6 +28,6 @@ observation has been emitted. A claim that the label matches its source text does not pass this check; the result must identify the canonical record used to establish the join key. 5. `/quality improve` does not call observation wiring complete while intended - mapped proof is unmatched or observations are ambiguous. -6. Any mechanical repair preserves check meaning, proof behavior, and + mapped verification methods are unmatched or observations are ambiguous. +6. Any mechanical repair preserves check meaning, verification behavior, and human-owned fields.