From 3287182e9b9fe168ed7a7a50e34c7b45079dcba4 Mon Sep 17 00:00:00 2001 From: Obvious Date: Thu, 17 Sep 2026 20:22:05 +0000 Subject: [PATCH 1/2] =?UTF-8?q?feat(writing-domain):=20first-party=20writi?= =?UTF-8?q?ng=20domain=20=E2=80=94=20six=20rules,=20compose=20intent,=20co?= =?UTF-8?q?mplete=20advice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Register the writing domain through the same registry path user domains take: one globally unique fallback intent ("compose") and six rules covering the eval vocabulary's six gaps — audience, purpose, structure/format, source material, context, completeness — at the vocabulary-pinned severities (high/high/medium/high/medium/low). All term matching goes through the shared word-boundary matcher (inputguard.matching, PR #6). Rules follow the trigger-and-satisfy shape; the source-material rule fires only when existing text is referenced but not provided. Recommendations and follow-up entries ship for all six gaps in the same change, keeping the registry completeness invariant whole. Co-authored-by: Kalisetti Nihanth Naidu --- inputguard/followups.py | 21 ++ inputguard/recommender.py | 36 +++ inputguard/rules/__init__.py | 22 +- inputguard/rules/writing.py | 431 +++++++++++++++++++++++++++++++++++ 4 files changed, 505 insertions(+), 5 deletions(-) create mode 100644 inputguard/rules/writing.py diff --git a/inputguard/followups.py b/inputguard/followups.py index ff8e590..a1fba74 100644 --- a/inputguard/followups.py +++ b/inputguard/followups.py @@ -94,6 +94,27 @@ "completion criteria": ( "What does 'done' look like — what should you be able to do when the feature works?", ), + "audience": ( + "Who is going to read this — a manager, your team, clients, or the public?", + "How familiar will readers be with the topic?", + ), + "purpose": ( + "What should this piece accomplish — inform, persuade, announce, or request something?", + ), + "structure/format": ( + "How long should it be, and how should it be organized — bullets, sections, or flowing prose?", + ), + "source material": ( + "Where is the text to work from — can you paste it or attach the file?", + "Should I preserve its structure, or can I reorganize it freely?", + ), + "context": ( + "What is the topic or situation this piece should cover?", + ), + "completeness": ( + "Is there anything it must include — specific points, names, or numbers?", + "Any hard limits, like a word count or a required section?", + ), } # Slot names a template may reference. A template naming anything else is a diff --git a/inputguard/recommender.py b/inputguard/recommender.py index 56c42bf..d17dc50 100644 --- a/inputguard/recommender.py +++ b/inputguard/recommender.py @@ -112,6 +112,42 @@ "what_to_provide": "Describe what done looks like. For example: 'the feature is complete when a user can type in the search box and see matching results appear within 200ms' or 'done means the user receives an email notification within 30 seconds of placing an order'. This prevents the AI from stopping too early or going too far.", "why_it_matters": "Without a clear definition of done, the AI may ship a half-finished feature or over-engineer something well past what you needed.", }, + "audience": { + "gap": "audience", + "what_is_missing": "You haven't said who will read this.", + "what_to_provide": "Name the reader and what they already know. For example: 'for the executive team, keep it high-level', 'for beginners who have never used the tool', 'for the client stakeholders, no jargon'. Even one phrase like 'for my manager' sharpens the tone and level of detail.", + "why_it_matters": "The same topic reads completely differently for a CEO, a new hire, or a customer. Without a named reader, the AI aims the piece at nobody in particular.", + }, + "purpose": { + "gap": "purpose", + "what_is_missing": "You haven't said what this piece should accomplish.", + "what_to_provide": "State the goal in one phrase: 'to persuade the steering committee to fund Q1 headcount', 'to announce the launch', 'to explain why the deadline moved'. A goal like 'convince', 'inform', or 'ask for' is enough to aim the writing.", + "why_it_matters": "Informing and persuading lead to different structures, evidence, and tone. Without a goal, the AI produces a generic piece that does neither well.", + }, + "structure/format": { + "gap": "structure/format", + "what_is_missing": "You haven't said how long the piece should be or how it should be organized.", + "what_to_provide": "Give a length and a shape. For example: 'under 300 words', 'one page', 'bullets with a short intro', 'three sections: situation, options, recommendation'. Any constraint — even just 'keep it short' — works.", + "why_it_matters": "Without length or organization guidance, the AI picks its own shape. You will often get a bloated draft you then have to cut down yourself.", + }, + "source material": { + "gap": "source material", + "what_is_missing": "You asked to work on existing text but didn't provide it.", + "what_to_provide": "Paste the text you want reworked — the draft, notes, or paragraph — directly into the message, or point to where it lives ('the outline is at the bottom'). Include any version details that matter.", + "why_it_matters": "The AI cannot see text that isn't in the message. Without it, you get a generic rewrite of an imaginary document instead of an improvement to yours.", + }, + "context": { + "gap": "context", + "what_is_missing": "You haven't said what the piece is about or what situation it responds to.", + "what_to_provide": "Name the subject and the situation. For example: 'about remote work for our company blog', 'regarding the Q3 roadmap', 'the email should tell the team the migration finished'. One sentence of background is enough.", + "why_it_matters": "Without a topic or situation, there is nothing to write about. The AI either invents one or asks you everything you could have said up front.", + }, + "completeness": { + "gap": "completeness", + "what_is_missing": "You haven't listed anything the piece must include.", + "what_to_provide": "Name the must-haves: 'include the headline, a quote from the CEO, and pricing', 'must cover current costs, risks, and the timeline', 'mention the new ship date'. Also note any hard limits, like a word count.", + "why_it_matters": "Must-have details left out of the request get left out of the draft. Naming them up front saves a second pass to work them in.", + }, } diff --git a/inputguard/rules/__init__.py b/inputguard/rules/__init__.py index fc32218..203c47f 100644 --- a/inputguard/rules/__init__.py +++ b/inputguard/rules/__init__.py @@ -1,9 +1,11 @@ -"""Built-in rule modules and the registry wiring for the coding domain. +"""Built-in rule modules and the registry wiring for the coding and +writing domains. -Importing this package registers the coding domain — its intent signals and -all 19 built-in rules — through the exact same registry path a user rule -takes. The v0.2 ``run_*_rules`` functions stay exported for backward -compatibility, but the analyzer dispatches through the registry now. +Importing this package registers both first-party domains — their intent +signals and all built-in rules — through the exact same registry path a +user rule takes. The v0.2 ``run_*_rules`` functions stay exported for +backward compatibility, but the analyzer dispatches through the registry +now. """ from inputguard.detector import ( @@ -22,6 +24,7 @@ OPTIMIZATION_RULES, run_optimization_rules, ) +from inputguard.rules.writing import WRITING_RULES, WRITING_SIGNALS __all__ = [ "run_coding_rules", @@ -45,3 +48,12 @@ *FEATURE_RULES, ), ) + +# The writing domain: a single fallback intent ("compose" — globally unique; +# no coding intent name is reused, so no cross-domain rule leakage) and the +# six first-party writing rules, registered through the same path. +REGISTRY.register_domain( + "writing", + WRITING_SIGNALS, + rules=WRITING_RULES, +) diff --git a/inputguard/rules/writing.py b/inputguard/rules/writing.py new file mode 100644 index 0000000..a0b8007 --- /dev/null +++ b/inputguard/rules/writing.py @@ -0,0 +1,431 @@ +"""First-party writing domain rules. + +The writing domain covers essays, emails, documents, and social posts. +Its gap vocabulary (pinned by the eval set's ``gap_vocabulary`` sheet) is +six gaps, one rule each: + +============ ========== ========================== +Gap Severity Rule +============ ========== ========================== +audience high missing_audience +purpose high missing_purpose +structure/ medium missing_structure_format +format +source high missing_source_material +material +context medium missing_writing_context +completeness low missing_completeness +============ ========== ========================== + +Rules follow the established trigger-and-satisfy shape: each fires only on +writing tasks and only when the gap's satisfier is absent from the prompt. + +Design note — one intent, not many. The six gaps apply to any writing task +(drafting, rewriting, feedback), so the domain declares a single fallback +intent (``compose``) instead of fragmenting into sub-intents whose inputs +would silently skip the rules. The compose-vs-revise discrimination the +coding domains put in intent signals lives here inside the rules that need +it (``missing_source_material`` fires only when the task references +existing material). + +All term matching goes through the shared word-boundary matcher +(``inputguard.matching``): ``"write"`` matches ``"writing"`` but not +``"written in Go"``, and ``"board"`` matches ``"the board"`` but not +``"keyboard"``. Phrase terms match adjacent words; the non-adjacent token +fallback is deliberately left off — these are new rules, so there is no +v0.2 verdict to preserve. +""" + +from __future__ import annotations + +import re +from typing import List, Optional, Tuple + +from inputguard.matching import contains_any +from inputguard.registry import register_rule +from inputguard.types import RuleFinding + + +# A writing task is present when the prompt names a writing artifact or a +# writing action. Every rule gates on this before looking at its own gap — +# it keeps coding prompts analyzed under domain="writing" (a user error) +# from producing misleading findings. +WRITING_TASK_TERMS = { + "write", "draft", "compose", + "blog", "post", "email", "memo", "letter", "essay", "resume", + "proposal", "paragraph", "article", "recap", "summary", "summaries", + "announcement", "press release", "cover letter", "outline", + "linkedin", "newsletter", "bio", "story", "tweet", "thread", + "note", "notes", "report", "document", "review", + "rewrite", "revise", "reword", "rephrase", "proofread", "polish", + "condense", "shorten", "tighten", "edit", +} + +# The named readership that satisfies the audience gap. Reader nouns are +# matched anywhere in the text ("admissions officers are the readers" has +# no "for" prefix, and it must satisfy). +AUDIENCE_TERMS = { + "audience", "readers", "reader", "readership", + "beginners", "beginner", "newcomers", + "manager", "managers", "my manager", "your manager", "my boss", + "supervisor", "supervisors", + "executives", "executive team", "leadership", "leadership team", + "board", "board members", "steering committee", "committee", + "stakeholders", "stakeholder", + "clients", "client", "customers", "customer", + "team", "my team", "engineering team", "engineers", "engineer", + "developers", "developer", + "journalists", "journalist", "press", "media", + "recruiters", "recruiter", "hiring manager", "hiring managers", + "admissions officers", "admissions", "officers", + "students", "student", "teachers", "teacher", "professors", + "professor", "instructors", "instructor", + "investors", "investor", "shareholders", "shareholder", + "subscribers", "subscriber", "followers", "follower", + "colleagues", "colleague", "coworkers", "teammates", "peers", + "donors", "donor", "members", "community", + "applicants", "applicant", "candidates", "candidate", + "reviewers", "reviewer", "editor", "editors", + "users", "user base", "general public", +} + +# A stated goal that satisfies the purpose gap ("announcing the Q3 +# roadmap", "asking for budget", "so that nobody reruns the old pipeline"). +PURPOSE_TERMS = { + "to convince", "convince", "convincing", "persuade", "persuading", + "persuasive", + "announce", "announcing", "announced", "announcement", + "request", "requesting", "requested", + "ask for", "asking for", "asks for", + "goal is", "goal of", "aim is", "aimed at", "purpose is", + "objective is", + "so that", "in order to", + "recap", + "lead with", "leads with", + "explain", "explaining", "explains", + "apologize", "apologizing", "apology", + "thank", "thanking", "thanks", + "celebrate", "celebrating", "celebration", + "invite", "inviting", "invitation", + "pitch", "pitching", "propose", "proposing", + "inform", "informing", "update", "updating", + "introduce", "introducing", + "complain", "complaining", "complaint", + "seeking", "seek", "justify", "justifying", + "follow up", "follow-up", "following up", + "respond", "responding", "reply", + "remind", "reminding", "reminder", + "promote", "promoting", "marketing", +} + +# Length or organization guidance that satisfies the structure/format gap. +# Bare artifact types ("blog post") do not satisfy — the vocabulary is +# explicit about that — so "page" and "word" only match in count forms. +STRUCTURE_TERMS = { + "bullet", "bullets", "bullet point", "bullet points", + "sections", "section", "headings", "heading", "subheadings", + "subheading", "numbered", "numbered list", + "concise", "short", "shorter", "brief", "condensed", + "length", "word count", "word limit", "page limit", + "table format", "as a table", "in a table", + "paragraph", "paragraphs", +} +# Count forms: "300 words", "one page", "two pages", "under 150 words", +# "3 bullets", "2 sections" — including hyphenated modifiers ("300-word"). +_STRUCTURE_COUNT_PATTERNS = ( + r"\b\d+\s*[-\s]?\s*words?\b", + r"\b(?:one|two|three|four|five|six|ten|single|half|\d+)\s*[-\s]?\s*pages?\b", + r"\b\d+\s*[-\s]?\s*(?:paragraphs?|bullets?|sections?|slides?)\b", +) + +# The task references existing material it wants worked on. Fires the +# source-material rule; the gap is then satisfied only if the material is +# actually provided. +MATERIAL_REFERENCE_TERMS = { + "rewrite", "revise", "reword", "rephrase", "proofread", "polish", + "improve", "condense", "shorten", "tighten", "trim", "expand", + "fix up", "clean up", "restructure", "reorganize", + "summarize", "summarise", "edit", "edited", "editing", "edits", + "turn the", "turn my", "turn this", + "my resume", "my essay", "my draft", "my outline", "my paragraph", + "my summary", "my cover letter", "my letter", "my email", "my post", + "my blog", "my notes", "my bio", "my article", "my proposal", + "my story", "my memo", "my document", "my report", + "this paragraph", "this essay", "this draft", "this outline", + "this email", "this document", "this summary", "this post", + "this blog", "this letter", "this resume", "this memo", "this note", + "this report", "this article", + "the essay", "the draft", "the outline", "the paragraph", + "the notes", "the meeting notes", "the resume", "the cover letter", + "the summary", "the document", "the manuscript", + "the survey responses", "the bullet outline", +} +# The material is actually provided: pasted into the prompt, attached, or +# pointed at concretely. +MATERIAL_PROVIDED_TERMS = { + "pasted", "attached", "below", "above", "at the bottom", "at the end", + "copied", "the following", "as follows", "included", "quoted", + "in the attachment", "in the document", "in the file", "the transcript", +} + +# A named subject or situation that satisfies the context gap: explicit +# context markers, topic introducers, the artifact itself, or a concrete +# situation noun ("the migration", "the Q3 roadmap", "the vendor"). +CONTEXT_TERMS = { + "context", "background", "regarding", "about", "on the topic of", + "subject:", "topic:", "re:", + "blog", "post", "email", "memo", "letter", "essay", "resume", + "proposal", "paragraph", "article", "recap", "summary", + "announcement", "press release", "cover letter", "outline", + "linkedin", "newsletter", "bio", "story", "tweet", + "note", "notes", "report", "document", "draft", "review", + "migration", "launch", "release", "deadline", "roadmap", "project", + "initiative", "campaign", "rollout", "transition", "incident", + "outage", "meeting", "event", "acquisition", "merger", "reorg", + "vendor", "offsite", "layoff", "hiring", "product", "issue", + "delay", "expansion", "funding", "series a", "restructure", +} + +# Enumerated required content or constraints that satisfy the +# completeness gap ("must cover current costs", "mention the new ship +# date", "include the headline", "max 300 words"). Bare "cover" is +# deliberately absent — "cover letter" must not satisfy. +COMPLETENESS_TERMS = { + "include", "including", "included", "includes", + "must cover", "should cover", "needs to cover", "covering", + "mention", "mentions", "mentioning", "mentioned", + "highlight", "highlighting", "highlights", + "emphasize", "emphasizing", "emphasise", "emphasising", + "such as", "specifically", "namely", "in particular", + "at least", "at minimum", + "make sure", "be sure to", "don't forget", "do not forget", + "required", "require", "requires", "requirements", + "must have", "must-have", "needs to have", "needs to include", + "max", "maximum", "at most", "no more than", "stay under", + "within", "limit", "up to", +} + + +def _normalize(text: str) -> str: + """Defensive re-normalization for direct callers of the check functions.""" + return re.sub(r"\s+", " ", text.strip().lower()) + + +def check_missing_audience(text: str) -> Optional[RuleFinding]: + text = _normalize(text) + if not contains_any(text, WRITING_TASK_TERMS): + return None + if contains_any(text, AUDIENCE_TERMS): + return None + return RuleFinding( + code="missing_audience", + message="Writing task detected but no audience or reader is specified.", + severity="high", + gap="audience", + ) + + +def check_missing_purpose(text: str) -> Optional[RuleFinding]: + text = _normalize(text) + if not contains_any(text, WRITING_TASK_TERMS): + return None + if contains_any(text, PURPOSE_TERMS): + return None + return RuleFinding( + code="missing_purpose", + message="Writing task detected but the goal — what the piece should accomplish — is not stated.", + severity="high", + gap="purpose", + ) + + +def _has_structure_signal(text: str) -> bool: + if contains_any(text, STRUCTURE_TERMS): + return True + return any(re.search(pattern, text) for pattern in _STRUCTURE_COUNT_PATTERNS) + + +def check_missing_structure_format(text: str) -> Optional[RuleFinding]: + text = _normalize(text) + if not contains_any(text, WRITING_TASK_TERMS): + return None + if _has_structure_signal(text): + return None + return RuleFinding( + code="missing_structure_format", + message="Writing task detected but no length or organization guidance is provided.", + severity="medium", + gap="structure/format", + ) + + +def check_missing_source_material(text: str) -> Optional[RuleFinding]: + text = _normalize(text) + if not contains_any(text, WRITING_TASK_TERMS): + return None + if not contains_any(text, MATERIAL_REFERENCE_TERMS): + return None + if contains_any(text, MATERIAL_PROVIDED_TERMS): + return None + return RuleFinding( + code="missing_source_material", + message="Existing material is referenced but not provided — paste or attach the text to work from.", + severity="high", + gap="source material", + ) + + +def check_missing_writing_context(text: str) -> Optional[RuleFinding]: + text = _normalize(text) + if not contains_any(text, WRITING_TASK_TERMS): + return None + if contains_any(text, CONTEXT_TERMS): + return None + return RuleFinding( + code="missing_writing_context", + message="Writing task detected but the subject or situation is not named.", + severity="medium", + gap="context", + ) + + +def check_missing_completeness(text: str) -> Optional[RuleFinding]: + text = _normalize(text) + if not contains_any(text, WRITING_TASK_TERMS): + return None + if contains_any(text, COMPLETENESS_TERMS): + return None + return RuleFinding( + code="missing_completeness", + message="Writing task detected but no required content or constraints are specified.", + severity="low", + gap="completeness", + ) + + +# Registry adapters: the check functions above stay the single home of the +# rule logic; these classes expose it through the v0.3 Rule protocol and +# register it through the same path a user rule takes. + + +@register_rule +class MissingAudienceRule: + """Registry adapter for check_missing_audience.""" + + id = "missing_audience" + domain = "compose" + severity = "high" + gap = "audience" + + def check(self, text: str) -> Optional[RuleFinding]: + return check_missing_audience(text) + + +@register_rule +class MissingPurposeRule: + """Registry adapter for check_missing_purpose.""" + + id = "missing_purpose" + domain = "compose" + severity = "high" + gap = "purpose" + + def check(self, text: str) -> Optional[RuleFinding]: + return check_missing_purpose(text) + + +@register_rule +class MissingStructureFormatRule: + """Registry adapter for check_missing_structure_format.""" + + id = "missing_structure_format" + domain = "compose" + severity = "medium" + gap = "structure/format" + + def check(self, text: str) -> Optional[RuleFinding]: + return check_missing_structure_format(text) + + +@register_rule +class MissingSourceMaterialRule: + """Registry adapter for check_missing_source_material.""" + + id = "missing_source_material" + domain = "compose" + severity = "high" + gap = "source material" + + def check(self, text: str) -> Optional[RuleFinding]: + return check_missing_source_material(text) + + +@register_rule +class MissingWritingContextRule: + """Registry adapter for check_missing_writing_context.""" + + id = "missing_writing_context" + domain = "compose" + severity = "medium" + gap = "context" + + def check(self, text: str) -> Optional[RuleFinding]: + return check_missing_writing_context(text) + + +@register_rule +class MissingCompletenessRule: + """Registry adapter for check_missing_completeness.""" + + id = "missing_completeness" + domain = "compose" + severity = "low" + gap = "completeness" + + def check(self, text: str) -> Optional[RuleFinding]: + return check_missing_completeness(text) + + +WRITING_RULES: Tuple[type, ...] = ( + MissingAudienceRule, + MissingPurposeRule, + MissingStructureFormatRule, + MissingSourceMaterialRule, + MissingWritingContextRule, + MissingCompletenessRule, +) + +# The single fallback intent: every writing-domain input is a composition +# task. Exactly one empty-terms intent is what the registry requires. +WRITING_SIGNALS: Tuple[Tuple[str, tuple], ...] = ( + ("compose", ()), +) + +# Findings this domain can emit, for the completeness invariant. +WRITING_GAPS: Tuple[str, ...] = ( + "audience", + "purpose", + "structure/format", + "source material", + "context", + "completeness", +) + + +def run_writing_rules(text: str) -> List[RuleFinding]: + """Run every writing rule directly (mirror of the coding run_* helpers).""" + findings: List[RuleFinding] = [] + seen = set() + for check in ( + check_missing_audience, + check_missing_purpose, + check_missing_structure_format, + check_missing_source_material, + check_missing_writing_context, + check_missing_completeness, + ): + result = check(text) + if result and result.code not in seen: + findings.append(result) + seen.add(result.code) + return findings From 9c37057bad058357f69a0b95bbdcf42468519e91 Mon Sep 17 00:00:00 2001 From: Obvious Date: Thu, 17 Sep 2026 20:22:05 +0000 Subject: [PATCH 2/2] test(writing-domain): per-rule, completeness, integration, no-leakage, and eval-label coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five layers: per-rule fire/silent behavior; the writing-domain completeness invariant (every writing gap has a four-key recommendation and one or two follow-up questions); analyze() integration end to end (underspecified -> needs_clarification at score 30, fully specified -> ready, partially specified -> usable_with_warnings); no-leakage in both directions at registry and analyze level; and the 14 labeled English writing rows from eval/cases.csv pinned as a calibration net. The exact-set registry expectations extend from 19 coding rules to 25 rules across two domains — the guards are strengthened, not weakened. Co-authored-by: Kalisetti Nihanth Naidu --- tests/test_registry.py | 17 +- tests/test_writing_domain.py | 303 +++++++++++++++++++++++++++++++++++ 2 files changed, 317 insertions(+), 3 deletions(-) create mode 100644 tests/test_writing_domain.py diff --git a/tests/test_registry.py b/tests/test_registry.py index e60b294..4b74cad 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -45,6 +45,13 @@ "missing_existing_stack", "missing_feature_scope", "missing_completion_criteria", + # First-party writing domain (6 rules). + "missing_audience", + "missing_purpose", + "missing_structure_format", + "missing_source_material", + "missing_writing_context", + "missing_completeness", } _V02_INTENT_RUNNERS = { @@ -85,11 +92,13 @@ def registry_isolation(): REGISTRY._origins.update(origins_before) -# --- the 19 built-ins dogfood the registry path --------------------------- +# --- the 25 built-ins dogfood the registry path --------------------------- def test_all_builtin_rules_registered_through_registry(): - assert len(REGISTRY.rule_ids()) == 19 + # 19 coding + 6 writing built-ins. The exact-set guard is the point: + # a rule that stops registering (or an unregistered stray) fails here. + assert len(REGISTRY.rule_ids()) == 25 assert set(REGISTRY.rule_ids()) == EXPECTED_BUILTIN_IDS for rule in REGISTRY.rules(): assert isinstance(rule, Rule) @@ -97,7 +106,9 @@ def test_all_builtin_rules_registered_through_registry(): def test_coding_domain_registered_with_priority_signals(): - assert REGISTRY.domain_names() == ("coding",) + # Both first-party domains register at import: coding first, writing + # second. + assert REGISTRY.domain_names() == ("coding", "writing") chain = REGISTRY.get_domain_signals("coding") assert [intent for intent, _ in chain] == [ "debug", diff --git a/tests/test_writing_domain.py b/tests/test_writing_domain.py new file mode 100644 index 0000000..ca4e0c9 --- /dev/null +++ b/tests/test_writing_domain.py @@ -0,0 +1,303 @@ +"""First-party writing domain tests. + +Five layers, matching the acceptance criteria: + +- per-rule behavior: each of the six writing rules fires on a gap-bearing + writing prompt and stays silent when that gap is satisfied (or when the + prompt is not a writing task at all); +- the writing-domain completeness invariant: every registered writing gap + has a four-key recommendation entry and one or two curated follow-up + questions, and the intent the rules bind to ("compose") is globally + unique — no coding intent reuses it; +- analyze() integration end to end: an underspecified writing prompt comes + back needs_clarification with the right gaps, severities, score, and + follow-ups; a fully specified one comes back ready; +- no leakage in either direction, at the registry level and through the + public analyze() path: writing codes never appear in coding analyses and + coding codes never appear in writing ones; +- the labeled English writing rows in eval/cases.csv match their labels — + the domain's calibration net. (The FP/FN measurement harness stays the + source of truth for rates; this pins the rows the gap_vocabulary sheet + defines.) + +All existing tests are untouched — this file only adds. +""" + +from __future__ import annotations + +import csv +from pathlib import Path + +import pytest + +from inputguard import InputGuard +from inputguard.followups import _FOLLOW_UP_QUESTIONS +from inputguard.recommender import _RECOMMENDATIONS +from inputguard.registry import REGISTRY +from inputguard.rules.writing import ( + WRITING_GAPS, + WRITING_RULES, + check_missing_audience, + check_missing_completeness, + check_missing_purpose, + check_missing_source_material, + check_missing_structure_format, + check_missing_writing_context, + run_writing_rules, +) + +EVAL_CASES = Path(__file__).resolve().parent.parent / "eval" / "cases.csv" + + +# --- per-rule behavior: fires on the gap, silent when satisfied -------------- + + +@pytest.mark.parametrize( + "check, fires_on, silent_on", + [ + ( + check_missing_audience, + "Write a blog post about remote work", + "Write a blog post about remote work for the engineering team", + ), + ( + check_missing_purpose, + "Write a blog post about remote work", + "Write a blog post announcing our new remote-work policy", + ), + ( + check_missing_structure_format, + "Write a blog post about remote work", + "Write a short blog post about remote work", + ), + # Fires only when existing material is referenced but not provided; + # a rewrite-with-text (gap satisfied) and a bare drafting prompt + # (no trigger) are both silent. + ( + check_missing_source_material, + "Rewrite my resume summary", + "Rewrite the resume summary I pasted below", + ), + ( + check_missing_writing_context, + "Write something short for tomorrow", + "Write a blog post about remote work", + ), + ( + check_missing_completeness, + "Write a blog post about remote work", + "Write a blog post about remote work and include the headline stat", + ), + ], +) +def test_rule_fires_on_gap_and_stays_silent_when_satisfied(check, fires_on, silent_on): + finding = check(fires_on) + assert finding is not None, f"{check.__name__} must fire on {fires_on!r}" + + assert check(silent_on) is None, f"{check.__name__} must stay silent on {silent_on!r}" + + +def test_not_a_writing_task_produces_no_findings(): + # Coding prompts analyzed in the writing domain (a user error) must not + # produce misleading writing findings — the writing-task gate blocks + # every rule. + for text in ( + "fix the login crash", + "my endpoint returns 500 instead of 200", + "explain async", + ): + assert run_writing_rules(text) == [], f"expected no findings for {text!r}" + + +def test_every_writing_rule_binds_to_the_compose_intent_with_vocab_severity(): + # The gap_vocabulary sheet pins the severities; the intent must be + # "compose" for every rule (the single fallback intent). + expected = { + "missing_audience": "high", + "missing_purpose": "high", + "missing_structure_format": "medium", + "missing_source_material": "high", + "missing_writing_context": "medium", + "missing_completeness": "low", + } + for rule in WRITING_RULES: + assert rule.domain == "compose" + assert rule.severity == expected[rule.id] + assert rule.gap is not None + + +# --- the writing-domain completeness invariant ------------------------------ + + +def _writing_registry_rules(): + writing_ids = {rule.id for rule in WRITING_RULES} + return [r for r in REGISTRY.rules() if r.id in writing_ids] + + +def test_every_registered_writing_rule_has_a_recommendation_and_follow_ups(): + # Mirrors the global invariant in test_followups.py, scoped to the + # writing domain: a writing rule can ship only with complete advice. + for rule in _writing_registry_rules(): + assert rule.gap in _RECOMMENDATIONS, f"gap {rule.gap!r} has no recommendation entry" + assert rule.gap in _FOLLOW_UP_QUESTIONS, f"gap {rule.gap!r} has no follow-up question" + assert 1 <= len(_FOLLOW_UP_QUESTIONS[rule.gap]) <= 2 + + +def test_writing_gap_set_matches_the_domain_vocabulary(): + # The six gaps the eval set's gap_vocabulary sheet defines for writing + # are exactly what the rules declare — no more, no fewer. + declared = {rule.gap for rule in _writing_registry_rules()} + assert declared == set(WRITING_GAPS) + + +def test_compose_intent_is_globally_unique(): + # PR #8 contract: intent names are globally unique across domains. No + # coding intent may collide with the writing domain's single intent. + for intent in ("build", "debug", "optimization", "explanation", "feature"): + ids = [r.id for r in REGISTRY.rules_for_intent(intent)] + assert not writing_ids_in(ids), f"writing rules leaked into intent {intent!r}" + + +def writing_ids_in(ids): + writing_ids = {rule.id for rule in WRITING_RULES} + return [i for i in ids if i in writing_ids] + + +def test_rules_for_compose_are_exactly_the_writing_rules(): + assert {r.id for r in REGISTRY.rules_for_intent("compose")} == { + rule.id for rule in WRITING_RULES + } + + +# --- analyze() integration, end to end --------------------------------------- + + +def test_underspecified_writing_prompt_end_to_end(): + result = InputGuard().analyze("Write a blog post about remote work", domain="writing") + + assert result.status == "needs_clarification" + assert result.detected_intent == "compose" + assert set(result.gaps) == {"audience", "purpose", "structure/format", "completeness"} + # 100 - 25 (audience, high) - 25 (purpose, high) - 15 (structure, + # medium) - 5 (completeness, low) = 30. + assert result.clarity_score == 30 + severity_by_gap = {f.gap: f.severity for f in result.findings} + assert severity_by_gap == { + "audience": "high", + "purpose": "high", + "structure/format": "medium", + "completeness": "low", + } + # Advice and follow-ups are complete: one four-key recommendation and + # at least one question per gap, in gap order. + assert [r["gap"] for r in result.recommendations] == result.gaps + for rec in result.recommendations: + assert set(rec) == {"gap", "what_is_missing", "what_to_provide", "why_it_matters"} + assert len(result.follow_ups) >= len(result.gaps) + assert all(q.endswith("?") for q in result.follow_ups) + # English input: no degradation, honest full coverage. + assert result.detected_language == "en" + assert result.heuristic_coverage == "full" + assert result.degradation_note is None + + +def test_fully_specified_writing_prompt_is_ready(): + text = ( + "Turn my bullet outline into a full proposal for the steering " + "committee; goal is to win headcount for Q1; include budget, risks, " + "and milestones; the outline is at the bottom" + ) + result = InputGuard().analyze(text, domain="writing") + + assert result.status == "ready" + assert result.clarity_score == 100 + assert result.gaps == [] + assert result.findings == [] + assert result.follow_ups == [] + + +def test_partially_specified_writing_prompt_lands_in_warning_band(): + # Audience, purpose, and context are present; length/organization and + # required content are not. 100 - 15 (structure, medium) - 5 + # (completeness, low) = 80 -> usable_with_warnings in warning mode. + result = InputGuard().analyze( + "Write an email to the engineering team announcing the Q3 roadmap", + domain="writing", + ) + + assert result.status == "usable_with_warnings" + assert result.clarity_score == 80 + assert set(result.gaps) == {"structure/format", "completeness"} + + +# --- no leakage, either direction -------------------------------------------- + + +def test_coding_analyses_never_emit_writing_codes(): + guard = InputGuard() + for text in ( + "Write a blog post about remote work", + "Rewrite my resume summary", + "Write a short email to my team explaining why the deadline moved", + ): + result = guard.analyze(text) # default domain: coding + assert writing_ids_in(f.code for f in result.findings) == [], ( + f"writing codes leaked into a coding analysis of {text!r}" + ) + + +def test_writing_analyses_never_emit_coding_codes(): + coding_codes = {rule.id for rule in REGISTRY.rules_for_intent("build",)} | { + rule.id + for intent in ("build", "debug", "optimization", "explanation", "feature") + for rule in REGISTRY.rules_for_intent(intent) + } + guard = InputGuard() + for text in ( + "fix the login crash", + "make this faster", + "add auth to my app", + "explain async", + ): + result = guard.analyze(text, domain="writing") + leaked = [f.code for f in result.findings if f.code in coding_codes] + assert leaked == [], f"coding codes leaked into a writing analysis of {text!r}" + + +# --- the labeled eval rows (calibration net) --------------------------------- + + +def _writing_eval_rows(): + with open(EVAL_CASES, newline="", encoding="utf-8") as handle: + rows = [row for row in csv.DictReader(handle) if row["domain"] == "writing"] + assert rows, "eval/cases.csv must contain writing rows" + return rows + + +def _parse_multi_value(raw: str): + """Split a ``;``-separated eval field; ``none``/``n/a`` mean no entries.""" + value = (raw or "").strip() + if value.lower() in {"none", "n/a", ""}: + return [] + return [part.strip() for part in value.split(";") if part.strip()] + + +@pytest.mark.parametrize("row", _writing_eval_rows(), ids=lambda row: row["id"]) +def test_labeled_writing_rows_match_their_labels(row): + guard = InputGuard() + result = guard.analyze(row["text"], domain="writing") + + expected_gaps = _parse_multi_value(row["expected_gaps"]) + expected_severities = _parse_multi_value(row["expected_severities"]) + + assert set(result.gaps) == set(expected_gaps), ( + f"{row['id']}: gap set mismatch on {row['text']!r}" + ) + assert result.status == row["expected_status"], ( + f"{row['id']}: status mismatch on {row['text']!r}" + ) + severity_by_gap = {f.gap: f.severity for f in result.findings} + for gap, severity in zip(expected_gaps, expected_severities): + assert severity_by_gap.get(gap) == severity, ( + f"{row['id']}: severity mismatch for gap {gap!r} on {row['text']!r}" + )