diff --git a/CHANGELOG.md b/CHANGELOG.md index 30c5e29..657c64c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.5.0] - 2026-08-11 + +Closes the last silent-pass class in the flag checker: short flags were +never scanned. The interesting part is what a longer single-dash token +might mean — `-xzf`, `-name`, and `-j4` are three different things — so +unresolved readings degrade to a warning rather than risking a false error. + +### Added + +- **Short flags are checked.** Only `--long` forms were scanned, so every + `-v` in generated content was a silent pass. A single-letter short flag is + unambiguous — it is that flag or nothing — so an absent one is an error, + the same as a long flag. +- **Ambiguous short tokens degrade to a warning.** A longer single-dash + token has several readings: `-xzf` may be a cluster of three flags, + `-name` a single-dash long option (find, java, and friends), `-j4` a flag + with an attached value. Each reading is tried against `--help` — a cluster + verifies when every letter is a known flag, an attached value when the + leading flag is known — and only if none verifies is a finding emitted, as + a warning rather than an error. Splitting `-name` into four letter flags + that do not exist would have been the obvious way to get this wrong. +- **A dash followed by digits is not a flag.** `--threshold -5` reads `-5` + as the value it almost always is. The cost is that a numeric short flag + (`head -5`) goes unchecked; the alternative false-positives on every + negative number in a command line. + +### Fixed + +- **A short flag no longer verifies against a longer flag that contains it.** + `_flag_in_help` bounded a match on the trailing side only, so `-v` matched + the tail of `--v` and reported a flag the command does not have as + verified. Both sides are now bounded. (The trailing bound already stopped + `-v` matching inside `--verbose`; this closes the leading side.) + ## [0.4.0] - 2026-08-11 Closes the gap left open at 0.3.0: reference-style links were never diff --git a/README.md b/README.md index d409d1e..8b26a64 100644 --- a/README.md +++ b/README.md @@ -72,9 +72,17 @@ literal text, so the link does not exist. An *undefined shortcut* is ordinary prose (`the [3] case` is not a broken link) and is skipped, as are GFM footnotes, which share the same syntax. +Both `--long` and `-short` flags are checked. A single-letter short flag is +unambiguous, so an absent one is an error. A longer single-dash token is not: +`-xzf` may be a cluster of three flags, `-name` a single-dash long option, and +`-j4` a flag with an attached value. Each reading is tried, and if none +verifies the finding is a **warning** rather than an error — an ambiguous token +is unverifiable, not refuted. + ### Known limitations -- Short flags (`-v`) are not checked; only `--long` forms are. +- A dash followed by digits (`-5`) is read as a negative number, not a flag, + so a numeric short flag (`head -5`) is not checked. - A flag written as bare `` `--flag` `` in prose is attributed to the nearest preceding word, so it may degrade to a warning rather than resolve. - Counts are matched to a source by keyword overlap with the claim's diff --git a/pyproject.toml b/pyproject.toml index ec45139..31751d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "attune-verify" -version = "0.4.0" +version = "0.5.0" description = "Generation fact-checker for the attune-* family. Verifies named entities in LLM output actually exist — imports import, CLI flags are real, links resolve, counts match source." readme = {file = "README.md", content-type = "text/markdown"} requires-python = ">=3.10" diff --git a/src/attune_verify/__init__.py b/src/attune_verify/__init__.py index 93497cb..269ffce 100644 --- a/src/attune_verify/__init__.py +++ b/src/attune_verify/__init__.py @@ -23,7 +23,7 @@ raise_if_failed, ) -__version__ = "0.4.0" +__version__ = "0.5.0" __all__ = [ "verify", "VerifyContext", diff --git a/src/attune_verify/checkers/flags.py b/src/attune_verify/checkers/flags.py index 00ed6e5..f2d4182 100644 --- a/src/attune_verify/checkers/flags.py +++ b/src/attune_verify/checkers/flags.py @@ -11,9 +11,14 @@ # One inline code span (`mytool --flag`); fences are handled separately. _INLINE_CODE_RE = re.compile(r"`([^`\n]+)`") -# A flag token anywhere in code text. Stops before "=value"; the negative -# lookbehind keeps it from matching the tail of a longer flag or a "---" rule. -_FLAG_TOKEN_RE = re.compile(r"(? warning, never a silent + # pass — the same rule as a command with no --help. return Finding( kind=FindingKind.UNKNOWN_FLAG, - detail=f"Flag '{flag}' not found in '{cmd} --help'", + detail=( + f"Short flag '{flag}' could not be verified against " + f"'{cmd} --help' — not found whole, as a cluster of " + "single-letter flags, or as a flag with an attached value" + ), evidence=evidence, - severity="error", + severity="warning", ) + return Finding( + kind=FindingKind.UNKNOWN_FLAG, + detail=f"Flag '{flag}' not found in '{cmd} --help'", + evidence=evidence, + severity="error", + ) + + +def _is_ambiguous_short(flag: str) -> bool: + """True for a single-dash token longer than one letter. + + ``-v`` is unambiguous: it is that flag or nothing. ``-xzf`` is not — it + could be three flags, one flag, or a flag plus a value. + """ + return not flag.startswith("--") and len(flag) > 2 + + +def _alternate_reading(flag: str, help_text: str) -> Optional[List[str]]: + """Return the first alternate reading of a short flag that fully verifies. + + Only single-dash tokens have alternate readings. A cluster verifies when + EVERY letter is a known flag (``-xzf`` against ``-x -z -f``); an attached + value verifies when the leading flag is known (``-j4`` against ``-j``). + """ + if not _is_ambiguous_short(flag): + return None + body = flag[1:] + if body.isalpha(): + cluster = [f"-{letter}" for letter in body] + if all(_flag_in_help(part, help_text) for part in cluster): + return cluster + attached = _ATTACHED_VALUE_RE.fullmatch(flag) + if attached and _flag_in_help(f"-{attached.group(1)}", help_text): + return [f"-{attached.group(1)}"] return None @@ -103,10 +156,12 @@ def _flag_in_help(flag: str, help_text: str) -> bool: """Return True if flag appears in help as a whole token. A plain substring test gives false negatives: ``--ver`` would pass - because ``--verbose`` contains it. Require the flag not be immediately - followed by another flag character (word char or hyphen). + because ``--verbose`` contains it. Require the flag be bounded on BOTH + sides by a non-flag character — the trailing bound stops ``-v`` matching + inside ``--verbose``, and the leading bound stops it matching the tail of + ``--v``, which would verify a short flag the command does not have. """ - return re.search(re.escape(flag) + r"(?![\w-])", help_text) is not None + return re.search(r"(? str: diff --git a/tests/corpus/cases.py b/tests/corpus/cases.py index 20b59c8..c7d3789 100644 --- a/tests/corpus/cases.py +++ b/tests/corpus/cases.py @@ -380,6 +380,52 @@ def _py(code: str) -> str: content="Write it as `[text](target.md)` — see [the doc](docs/missing.md).", expected=(ExpectedFinding(FindingKind.DEAD_LINK, "docs/missing.md"),), ), + CorpusCase( + name="short_flag_unknown_flagged", + label="hallucinated", + # A single-char short flag is unambiguous: it is that flag or nothing, + # so an absent one is refuted, not merely unverifiable. + content="Run `mytool -q` to stay quiet.", + help_commands={"mytool": "Usage: mytool\n -v, --verbose Be loud\n"}, + expected=(ExpectedFinding(FindingKind.UNKNOWN_FLAG, "-q"),), + ), + CorpusCase( + name="clean_short_flag_cluster", + label="clean", + # '-xzf' is three flags run together; every letter is known. + content="Extract with `tar -xzf archive.tgz`.", + help_commands={"tar": "Usage: tar\n -x, --extract\n -z, --gzip\n -f, --file FILE\n"}, + ), + CorpusCase( + name="clean_short_flag_single_dash_long_name", + label="clean", + # find-style single-dash long options must not be split into a cluster + # of letters that do not exist. + content="Match with `find -name '*.py'`.", + help_commands={"find": "Usage: find\n -name PATTERN\n -type T\n"}, + ), + CorpusCase( + name="clean_short_flag_attached_value", + label="clean", + content="Build with `make -j4`.", + help_commands={"make": "Usage: make\n -j N, --jobs N\n"}, + ), + CorpusCase( + name="clean_negative_number_is_not_a_flag", + label="clean", + # A dash-number is far more often a value than a flag. + content="Filter with `mytool --threshold -5`.", + help_commands={"mytool": "Usage: mytool\n --threshold N Cutoff\n"}, + ), + CorpusCase( + name="evasion_short_flag_hiding_in_long_flag", + label="evasion", + # '-v' is a substring of '--verbose'; a boundary-free match would + # verify a short flag the command does not have. + content="Run `mytool -v` for detail.", + help_commands={"mytool": "Usage: mytool\n --verbose Be loud\n"}, + expected=(ExpectedFinding(FindingKind.UNKNOWN_FLAG, "-v"),), + ), CorpusCase( name="clean_reference_link_resolves", label="clean", diff --git a/tests/test_behavioral.py b/tests/test_behavioral.py index 8748193..8dc7ff3 100644 --- a/tests/test_behavioral.py +++ b/tests/test_behavioral.py @@ -267,6 +267,73 @@ def test_links_balanced_parens_in_target_are_not_truncated(tmp_path): # --------------------------------------------------------------------------- # Flag checker branches # --------------------------------------------------------------------------- +_TAR_HELP = {"tar": "Usage: tar\n -x, --extract\n -z, --gzip\n -f, --file F\n -v, --verbose\n"} + + +def _flags(content, help_map=None): + return check_flags(content, help_map or _TAR_HELP, frozenset()) + + +@pytest.mark.parametrize( + "content,help_map", + [ + ("`tar -v`", None), # single letter, known + ("`tar -xzf a.tgz`", None), # cluster, every letter known + ("`find -name '*.py'`", {"find": "Usage: find\n -name PAT\n -type T\n"}), + ("`make -j4`", {"make": "Usage: make\n -j N, --jobs N\n"}), # attached value + ], +) +def test_short_flag_readings_that_verify_are_clean(content, help_map): + assert _flags(content, help_map) == [] + + +def test_short_single_letter_flag_absent_is_an_error(): + # Unambiguous: '-q' is that flag or nothing. + findings = _flags("`tar -q`") + assert [(f.severity, "-q" in f.detail) for f in findings] == [("error", True)] + + +@pytest.mark.parametrize( + "content,help_map", + [ + ("`tar -xqf a.tgz`", None), # cluster with an unknown letter + ("`find -nope`", {"find": "Usage: find\n -name PAT\n"}), # single-dash long + ("`make -z9`", {"make": "Usage: make\n -j N\n"}), # attached value, unknown + ], +) +def test_ambiguous_short_flag_degrades_to_warning_not_error(content, help_map): + # '-xzf' may be a cluster, '-name' a single-dash long option, '-j4' a flag + # with an attached value. Calling an unresolved reading refuted would risk + # a false error on a real flag — unverifiable, so warn. + findings = _flags(content, help_map) + assert len(findings) == 1 + assert findings[0].severity == "warning" + + +@pytest.mark.parametrize( + "content", + [ + "`tar --threshold -5`", # negative number argument + "`cat -`", # bare dash (stdin) + "`tar -- -x`", # end-of-options separator + "Use `my-tool` for this.", # hyphenated word + "`tar -f /path/to-file`", # hyphen inside a path + ], +) +def test_tokens_that_are_not_short_flags(content): + # Each of these would become a false positive if the token pattern were + # any looser. '--threshold' is real here, so only non-flag tokens remain. + assert [f for f in _flags(content) if "-5" in f.detail or "-x" in f.detail] == [] + + +def test_short_flag_does_not_verify_against_a_longer_flag(): + # '-v' is a substring of '--verbose' and a suffix of '--v'; neither means + # the command has a '-v'. + for help_text in ("Usage: t\n --verbose Be loud\n", "Usage: t\n --v odd\n"): + findings = check_flags("`t -v`", {"t": help_text}, frozenset()) + assert [f.severity for f in findings] == ["error"], help_text + + def test_flag_present_in_cached_help_is_clean(): findings = check_flags( "Use `tool` `--verbose` now.",