diff --git a/CHANGELOG.md b/CHANGELOG.md index 4466700..30c5e29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.4.0] - 2026-08-11 + +Closes the gap left open at 0.3.0: reference-style links were never +resolved, so a dead target behind a label was a silent pass. Adding the +feature also meant closing three false-positive classes it would otherwise +have introduced — bracketed prose, GFM footnotes, and indented definitions. + +### Added + +- **Reference-style links are checked.** Only inline `[text](target)` links + were resolved, so every reference form was a silent pass — a dead target + behind a label went unreported. All three CommonMark forms now resolve + against the document's `[label]: target` definitions: full + `[text][label]`, collapsed `[text][]`, and shortcut `[text]`. Definitions + are matched case-insensitively with whitespace collapsed, may carry a + title or `` target, may be indented (including nested under + a list item), and the first definition of a repeated label wins. +- **An explicit reference with no definition is an error.** `[text][label]` + without a matching definition renders as literal text — the link does not + exist, which is a refuted claim rather than an unverifiable one. Findings + quote the reference form (`[text][label]`) rather than inline syntax the + reader would not find in their document. + +### Fixed + +- **Undefined shortcut references are not links.** `[3]` and `[TODO]` in + ordinary prose only become links when a definition exists, so an undefined + shortcut is skipped rather than flagged — the bracketed-prose false + positive that makes shortcut support worth having at all. +- **GFM footnotes are excluded.** A footnote shares reference syntax exactly: + `[^1]` against `[^1]: Sourced` parsed as a link whose target was the + footnote body, flagging `Sourced` as a missing file. The `^` namespace is + no longer read as a link reference. +- **Definitions inside code fences no longer define.** A renderer does not + read definitions out of a code block, so a reference that depends on one is + correctly reported as undefined. + ## [0.3.0] - 2026-08-11 **Beta.** The deterministic core is stable and the public API is now covered diff --git a/README.md b/README.md index c397266..d409d1e 100644 --- a/README.md +++ b/README.md @@ -64,10 +64,16 @@ protocol) is what beta covers: it will not change shape without a deprecation in a minor release. The LLM semantic layer is optional via the `[rag]` extra and is the least settled part of the surface. +Links are read from prose only. Inline (`[text](target)`) and all three +reference forms — full `[text][label]`, collapsed `[text][]`, and shortcut +`[text]` — resolve against the document's `[label]: target` definitions. An +explicit reference whose label is never defined is an error: it renders as +literal text, so the link does not exist. An *undefined shortcut* is ordinary +prose (`the [3] case` is not a broken link) and is skipped, as are GFM +footnotes, which share the same syntax. + ### Known limitations -- Reference-style links (`[text][ref]`) are not resolved — only inline - `[text](target)` links are checked. - Short flags (`-v`) are not checked; only `--long` forms are. - A flag written as bare `` `--flag` `` in prose is attributed to the nearest preceding word, so it may degrade to a warning rather than resolve. diff --git a/pyproject.toml b/pyproject.toml index 7eb9f6b..ec45139 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "attune-verify" -version = "0.3.0" +version = "0.4.0" description = "Generation fact-checker for the attune-* family. Verifies named entities in LLM output actually exist — imports import, CLI flags are real, links resolve, counts match source." readme = {file = "README.md", content-type = "text/markdown"} requires-python = ">=3.10" diff --git a/src/attune_verify/__init__.py b/src/attune_verify/__init__.py index da933d9..93497cb 100644 --- a/src/attune_verify/__init__.py +++ b/src/attune_verify/__init__.py @@ -23,7 +23,7 @@ raise_if_failed, ) -__version__ = "0.3.0" +__version__ = "0.4.0" __all__ = [ "verify", "VerifyContext", diff --git a/src/attune_verify/_extract.py b/src/attune_verify/_extract.py index f986636..3b42f06 100644 --- a/src/attune_verify/_extract.py +++ b/src/attune_verify/_extract.py @@ -18,11 +18,17 @@ class CodeFence: @dataclass class MarkdownLink: - """A markdown link extracted from content.""" + """A markdown link extracted from content. + + ``target`` is None only for a reference link whose label has no + definition: the reference names a definition that does not exist, which + is a claim in its own right and is reported as a dead link. + """ text: str - target: str + target: Optional[str] line: Optional[int] = None + label: Optional[str] = None # set when the link came from a [ref] form @dataclass @@ -56,6 +62,21 @@ class NumericClaim: # A markdown link target may carry a quoted/parenthesized title after the path # ('docs/a.md "Read me"') or wrap the path in . _LINK_TITLE_RE = re.compile(r"""^(\S+)\s+("[^"]*"|'[^']*'|\([^)]*\))$""") +# A link reference definition: '[label]: docs/a.md "Optional title"' leading a +# line. CommonMark caps the indent at three spaces (four starts an indented +# code block), but any indent is accepted here: a definition nested under a +# list item is real and common, and missing one turns its reference into an +# error-severity false positive — the costlier direction. Fenced blocks are +# masked before this runs; indented code blocks are not modelled anywhere in +# the extractor, so this stays consistent with the rest of it. +_LINK_DEF_RE = re.compile( + r"""^[ \t]*\[([^\]]+)\]:[ \t]*(\S+)(?:[ \t]+("[^"]*"|'[^']*'|\([^)]*\)))?[ \t]*$""", + re.MULTILINE, +) +# A reference link: full '[text][label]', collapsed '[text][]', or shortcut +# '[text]'. The second bracket group is None for the shortcut form and "" for +# the collapsed form — the two are treated differently when unresolved. +_REF_LINK_RE = re.compile(r"\[([^\]]+)\](?:\[([^\]]*)\])?") @dataclass @@ -190,15 +211,96 @@ def extract_links(content: str) -> List[MarkdownLink]: title (``docs/a.md "Read me"``) is stripped and ```` wrapping is removed, so checkers see only the path. """ - links = [] prose = _mask_code(content) + links = [] for match in _LINK_RE.finditer(prose): - line = prose[: match.start()].count("\n") + 1 links.append( MarkdownLink( text=match.group(1), target=_clean_link_target(match.group(2)), - line=line, + line=_line_of(prose, match.start()), + ) + ) + # Inline links are consumed first: their '[text]' would otherwise read as a + # shortcut reference. Definitions are consumed next for the same reason — + # '[ref]: docs/a.md' leads with something shaped exactly like one. + remaining = _mask_spans(prose, _LINK_RE) + definitions = _link_definitions(remaining) + remaining = _mask_spans(remaining, _LINK_DEF_RE) + links.extend(_reference_links(remaining, definitions)) + return links + + +def _line_of(text: str, offset: int) -> int: + """1-based line number of an offset. Masking preserves line breaks, so + this is the line in the original content.""" + return text[:offset].count("\n") + 1 + + +def _mask_spans(text: str, pattern: "re.Pattern[str]") -> str: + """Blank every match of pattern, preserving length and line breaks.""" + return pattern.sub(lambda m: _blank_like(m.group(0)), text) + + +def _blank_like(matched: str) -> str: + """Spaces of the same shape as the matched text, newlines kept.""" + return "".join("\n" if char == "\n" else " " for char in matched) + + +def _normalize_label(label: str) -> str: + """CommonMark label matching: case-insensitive, whitespace-collapsed.""" + return " ".join(label.split()).lower() + + +def _is_footnote_label(label: str) -> bool: + """True for the GFM footnote namespace, which is not a link reference. + + A footnote shares link-reference syntax exactly — ``[^1]`` against + ``[^1]: Sourced`` — so a short footnote body reads as a target and its + marker reads as a link to it, flagging "Sourced" as a missing file. + """ + return label.startswith("^") + + +def _link_definitions(prose: str) -> dict: + """Map normalized reference labels to their targets. + + A repeated label keeps the FIRST definition, as CommonMark specifies. + """ + definitions: dict = {} + for match in _LINK_DEF_RE.finditer(prose): + label = _normalize_label(match.group(1)) + if _is_footnote_label(label) or label in definitions: + continue + definitions[label] = _clean_link_target(match.group(2)) + return definitions + + +def _reference_links(prose: str, definitions: dict) -> List[MarkdownLink]: + """Resolve reference links against the document's definitions. + + Three forms: full ``[text][label]``, collapsed ``[text][]`` (label is the + text), and shortcut ``[text]`` (likewise). A shortcut whose label has no + definition is ordinary prose — "the [3] case" is not a broken link — so it + is skipped. The bracketed forms are an explicit reference: an undefined + label there renders literally instead of linking, so it is reported with + ``target=None`` rather than passing silently. + """ + links = [] + for match in _REF_LINK_RE.finditer(prose): + text, bracketed = match.group(1), match.group(2) + is_shortcut = bracketed is None + label = _normalize_label(bracketed if bracketed else text) + if _is_footnote_label(label): + continue + if label not in definitions and is_shortcut: + continue + links.append( + MarkdownLink( + text=text, + target=definitions.get(label), + line=_line_of(prose, match.start()), + label=label, ) ) return links diff --git a/src/attune_verify/checkers/links.py b/src/attune_verify/checkers/links.py index c4110b6..2da1f09 100644 --- a/src/attune_verify/checkers/links.py +++ b/src/attune_verify/checkers/links.py @@ -17,6 +17,9 @@ def check_links( """Verify markdown link targets exist relative to project_root. External URLs (http/https) are skipped — only local paths are checked. + A reference link whose label has no definition is reported directly: the + reference names a definition that does not exist, so there is no target + to look up. Args: links: Markdown links extracted from generated content. @@ -29,6 +32,23 @@ def check_links( findings: List[Finding] = [] for link in links: target = link.target + if target is None: + # An undefined reference does not render as a link at all — the + # raw '[text][label]' is what a reader sees. Refuted, not + # unverifiable, so this is an error like any other dead link. + findings.append( + Finding( + kind=FindingKind.DEAD_LINK, + detail=( + f"Link reference '[{link.label}]' is used but never " + "defined — no matching '[label]: target' definition" + ), + evidence=_evidence(link), + location=f"line {link.line}" if link.line else None, + severity="error", + ) + ) + continue # Skip external URLs and anchors-only if target.startswith(("http://", "https://", "mailto:", "#")): continue @@ -43,7 +63,7 @@ def check_links( detail=( f"Link '{target}' cannot be verified " "(no project_root in VerifyContext)" ), - evidence=f"[{link.text}]({link.target})", + evidence=_evidence(link), location=f"line {link.line}" if link.line else None, severity="warning", ) @@ -64,7 +84,7 @@ def check_links( detail=( f"Link '{target}' resolves outside project_root " "and cannot be verified" ), - evidence=f"[{link.text}]({link.target})", + evidence=_evidence(link), location=f"line {link.line}" if link.line else None, severity="warning", ) @@ -75,7 +95,7 @@ def check_links( Finding( kind=FindingKind.DEAD_LINK, detail=f"Link target '{path_part}' does not exist", - evidence=f"[{link.text}]({link.target})", + evidence=_evidence(link), location=f"line {link.line}" if link.line else None, severity="error", ) @@ -83,6 +103,17 @@ def check_links( return findings +def _evidence(link: MarkdownLink) -> str: + """Render the link the way it was written. + + A reference link quoted back as inline syntax would be evidence the reader + cannot find in their document, so reference forms keep their brackets. + """ + if link.label is not None: + return f"[{link.text}][{link.label}]" + return f"[{link.text}]({link.target})" + + def _resolve_target(root: Path, rel: str) -> Path: """Resolve a link target under root, honouring percent-encoding. diff --git a/tests/corpus/cases.py b/tests/corpus/cases.py index d9c6868..20b59c8 100644 --- a/tests/corpus/cases.py +++ b/tests/corpus/cases.py @@ -380,6 +380,48 @@ def _py(code: str) -> str: content="Write it as `[text](target.md)` — see [the doc](docs/missing.md).", expected=(ExpectedFinding(FindingKind.DEAD_LINK, "docs/missing.md"),), ), + CorpusCase( + name="clean_reference_link_resolves", + label="clean", + content="See [the guide][guide] for details.\n\n[guide]: docs/a.md\n", + files=("docs/a.md",), + ), + CorpusCase( + name="dead_reference_link_target_flagged", + label="hallucinated", + # Reference links were entirely unchecked before 0.4.0 — a dead target + # behind a label was a silent pass. + content="See [the guide][guide].\n\n[guide]: docs/missing.md\n", + expected=(ExpectedFinding(FindingKind.DEAD_LINK, "docs/missing.md"),), + ), + CorpusCase( + name="undefined_reference_label_flagged", + label="hallucinated", + # An explicit reference with no definition renders literally — the + # reader sees '[the guide][guide]', so the link never existed. + content="See [the guide][guide] for details.", + expected=(ExpectedFinding(FindingKind.DEAD_LINK, "guide"),), + ), + CorpusCase( + name="clean_shortcut_reference_resolves", + label="clean", + content="See [guide] for details.\n\n[guide]: docs/a.md\n", + files=("docs/a.md",), + ), + CorpusCase( + name="clean_bracketed_prose_is_not_a_link", + label="clean", + # An undefined SHORTCUT is ordinary prose, not a broken link — + # flagging it would false-positive on any bracketed text. + content="Handle the [3] case and the [TODO] items before shipping.", + ), + CorpusCase( + name="clean_footnote_is_not_a_link_reference", + label="clean", + # GFM footnotes share reference syntax exactly; a short footnote body + # read as a target and flagged 'Sourced' as a missing file. + content="The count is stable.[^1]\n\n[^1]: Sourced\n", + ), CorpusCase( name="clean_link_balanced_parens", label="clean", diff --git a/tests/test_behavioral.py b/tests/test_behavioral.py index 56faee1..8748193 100644 --- a/tests/test_behavioral.py +++ b/tests/test_behavioral.py @@ -492,6 +492,85 @@ def test_extract_links_span_containing_backticks_is_masked_whole(): assert [link.target for link in extract_links(content)] == ["real.md"] +# --------------------------------------------------------------------------- +# Reference-style links: the three CommonMark forms, definitions, footnotes +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "content", + [ + "See [the doc][guide].\n\n[guide]: docs/a.md\n", # full + "See [guide][].\n\n[guide]: docs/a.md\n", # collapsed + "See [guide].\n\n[guide]: docs/a.md\n", # shortcut + 'See [d][g].\n\n[g]: docs/a.md "Read me"\n', # definition with title + "See [d][g].\n\n[g]: \n", # angle-bracket definition + "See [The Guide][].\n\n[the guide]: docs/a.md\n", # case + whitespace + "See [d][g].\n\n [g]: docs/a.md\n", # 3-space indented definition + "1. See [d][g].\n\n [g]: docs/a.md\n", # definition under a list item + ], +) +def test_reference_link_forms_resolve_to_the_definition(content): + assert [link.target for link in extract_links(content)] == ["docs/a.md"] + + +def test_reference_link_first_definition_wins(): + content = "See [d][g].\n\n[g]: docs/first.md\n[g]: docs/second.md\n" + assert [link.target for link in extract_links(content)] == ["docs/first.md"] + + +def test_reference_link_line_number_is_the_reference_not_the_definition(): + content = "intro\nprose\n\nSee [d][g].\n\n[g]: docs/a.md\n" + assert [link.line for link in extract_links(content)] == [4] + + +def test_definition_inside_a_fence_does_not_define(): + # A renderer does not read definitions out of a code block, so the + # reference is genuinely undefined. + content = "See [d][g].\n\n```\n[g]: docs/a.md\n```\n" + links = extract_links(content) + assert [(link.target, link.label) for link in links] == [(None, "g")] + + +def test_undefined_explicit_reference_is_an_error(tmp_path): + links = extract_links("See [the doc][guide].") + findings = check_links(links, project_root=tmp_path) + assert len(findings) == 1 + assert findings[0].severity == "error" + assert "guide" in findings[0].detail + # Evidence must be the syntax the reader can find in their document. + assert findings[0].evidence == "[the doc][guide]" + + +def test_undefined_shortcut_reference_is_ordinary_prose(tmp_path): + # Flagging these would false-positive on any bracketed text. + links = extract_links("Handle the [3] case and the [TODO] items.") + assert links == [] + assert check_links(links, project_root=tmp_path) == [] + + +@pytest.mark.parametrize( + "content", + [ + "Claim.[^1]\n\n[^1]: Sourced\n", # short body reads as a target + "Claim.[^1]\n\n[^1]: Sourced from the docs\n", + ], +) +def test_footnotes_are_not_link_references(content, tmp_path): + assert extract_links(content) == [] + + +def test_reference_and_inline_links_coexist(tmp_path): + content = "See [a](docs/one.md) and [b][g].\n\n[g]: docs/two.md\n" + assert [link.target for link in extract_links(content)] == [ + "docs/one.md", + "docs/two.md", + ] + + +def test_reference_link_inside_a_code_span_is_skipped(): + content = "Write `[d][g]` here.\n\n[g]: docs/a.md\n" + assert extract_links(content) == [] + + def test_extract_links_captures_text_target_and_line(): links = extract_links("a\n[label](path/to.md)\n") assert len(links) == 1