diff --git a/.pdd/verification-profile-rotations.json b/.pdd/verification-profile-rotations.json index 9c699e077..935382820 100644 --- a/.pdd/verification-profile-rotations.json +++ b/.pdd/verification-profile-rotations.json @@ -657,6 +657,28 @@ "head_policy_sha256": "faf427d3891e0c4eb38a1d25d4d03f1fa4f24bcac642b7658bd65ceeaf52b953", "base_prompt_sha256": "882876b0dea9198c1fa9492806d85bfb088b393096f4f1a0543cc8f31f40fc30", "head_prompt_sha256": "a3eb3060fcf46f6534f1d483693c659b2a5d4068c5b309798c591b9166ee2704" + }, + { + "prompt_path": "pdd/prompts/generate_story_contract_LLM.prompt", + "language_id": "llm", + "from_requirement_id": "CONTRACT-SHA256:415e054e596f1b103e999a1f7dca848143c2d8be09b8b1723cd01ead1fe66fc6", + "to_requirement_id": "CONTRACT-SHA256:d51e79e13bb784b75b966728eaf3e56e8660d363d5c6729ede11b171af6e8e7c", + "policy_path": ".pdd/verification-profiles.json", + "base_policy_sha256": "faf427d3891e0c4eb38a1d25d4d03f1fa4f24bcac642b7658bd65ceeaf52b953", + "head_policy_sha256": "c5ec62e0907d4301cb26b8a63e56066fe57df2a48ceb35caa7d3b600dde6c49b", + "base_prompt_sha256": "415e054e596f1b103e999a1f7dca848143c2d8be09b8b1723cd01ead1fe66fc6", + "head_prompt_sha256": "d51e79e13bb784b75b966728eaf3e56e8660d363d5c6729ede11b171af6e8e7c" + }, + { + "prompt_path": "pdd/prompts/user_story_tests_python.prompt", + "language_id": "python", + "from_requirement_id": "CONTRACT-SHA256:1c467034344d9d87b8225995bc458bc8093e6759dd5c2eed8424b345f69a3ba7", + "to_requirement_id": "CONTRACT-SHA256:cbdfe0b435c9ee05f71a3279278b3e1d2d719803e8a28a0fe82b961e22d4e2d9", + "policy_path": ".pdd/verification-profiles.json", + "base_policy_sha256": "faf427d3891e0c4eb38a1d25d4d03f1fa4f24bcac642b7658bd65ceeaf52b953", + "head_policy_sha256": "c5ec62e0907d4301cb26b8a63e56066fe57df2a48ceb35caa7d3b600dde6c49b", + "base_prompt_sha256": "1c467034344d9d87b8225995bc458bc8093e6759dd5c2eed8424b345f69a3ba7", + "head_prompt_sha256": "cbdfe0b435c9ee05f71a3279278b3e1d2d719803e8a28a0fe82b961e22d4e2d9" } ], "requirement_rotation_retirements": [ diff --git a/.pdd/verification-profiles.json b/.pdd/verification-profiles.json index 4d52e26ae..539381bab 100644 --- a/.pdd/verification-profiles.json +++ b/.pdd/verification-profiles.json @@ -7421,7 +7421,7 @@ "prompt_path": "pdd/prompts/generate_story_contract_LLM.prompt", "language_id": "llm", "required_requirement_ids": [ - "CONTRACT-SHA256:415e054e596f1b103e999a1f7dca848143c2d8be09b8b1723cd01ead1fe66fc6" + "CONTRACT-SHA256:d51e79e13bb784b75b966728eaf3e56e8660d363d5c6729ede11b171af6e8e7c" ], "obligations": [ { @@ -7430,7 +7430,7 @@ "validator_id": "threshold-ed25519", "validator_config_digest": "threshold-ed25519-v1", "requirement_ids": [ - "CONTRACT-SHA256:415e054e596f1b103e999a1f7dca848143c2d8be09b8b1723cd01ead1fe66fc6" + "CONTRACT-SHA256:d51e79e13bb784b75b966728eaf3e56e8660d363d5c6729ede11b171af6e8e7c" ], "artifact_paths": [ "pdd/prompts/generate_story_contract_LLM.prompt" @@ -10351,7 +10351,7 @@ "prompt_path": "pdd/prompts/user_story_tests_python.prompt", "language_id": "python", "required_requirement_ids": [ - "CONTRACT-SHA256:1c467034344d9d87b8225995bc458bc8093e6759dd5c2eed8424b345f69a3ba7" + "CONTRACT-SHA256:cbdfe0b435c9ee05f71a3279278b3e1d2d719803e8a28a0fe82b961e22d4e2d9" ], "obligations": [ { @@ -10360,7 +10360,7 @@ "validator_id": "threshold-ed25519", "validator_config_digest": "threshold-ed25519-v1", "requirement_ids": [ - "CONTRACT-SHA256:1c467034344d9d87b8225995bc458bc8093e6759dd5c2eed8424b345f69a3ba7" + "CONTRACT-SHA256:cbdfe0b435c9ee05f71a3279278b3e1d2d719803e8a28a0fe82b961e22d4e2d9" ], "artifact_paths": [ "pdd/prompts/user_story_tests_python.prompt" diff --git a/pdd/prompts/generate_story_contract_LLM.prompt b/pdd/prompts/generate_story_contract_LLM.prompt index 7f873ab55..d0098fad8 100644 --- a/pdd/prompts/generate_story_contract_LLM.prompt +++ b/pdd/prompts/generate_story_contract_LLM.prompt @@ -31,6 +31,16 @@ {PRIMARY_PROMPTS} + + % Deterministically-resolved, importable module interfaces for the prompts + % already linked to this Story — computed by the tooling, not the model. + % Only a block with a `module:` line and a `functions:` list is usable as a + % behavioral test Entry Point; a block with no `module:` line means that + % prompt has no deterministic import binding (a CLI/Click interface, or one + % whose module path could not be resolved) and MUST NOT be used as one. + + {PRIMARY_PROMPT_INTERFACES} + % What the contract must do @@ -39,8 +49,10 @@ - Pull concrete, behavior-changing detail from the issue (per-source vs aggregate, `--json`, a `--threshold` exit code, "MUST NOT make an LLM call") into `## Covers`, `## Acceptance Criteria`, and `## Oracle`. Preserve distinctions precisely; do not collapse them into vague wording, and do not broaden a requirement beyond the issue/Story. - `## Covers` lists the named requirements/acceptance-criteria the story exercises (`- AC1: ` / `- R1: `), specific enough that removing that behavior fails validation. - `## Acceptance Criteria` are concrete Given/When/Then statements about observable behavior (prefer 2–5). - - `## Oracle` lists what decides pass/fail (error type, state transition, presence/absence of an external call, returned value shape). `## Non-Oracle` lists what must NOT matter. - - `## Negative Cases` lists forbidden outcomes (from MUST NOT / SHALL NOT rules or obviously-wrong behavior). + - `## Entry Point` is required. Copy `module:` and one name from that block's `functions:` list VERBATIM from a `` block that has a `module:` line — never invent, guess, or modify either string, and never select a block that has no `module:` line (it has no deterministic import binding — e.g. a CLI/Click command — and cannot be called directly). Use Python literals for `args` (a list) and `kwargs` (a dict); choose safe, deterministic arguments that exercise this Story. If NO linked prompt has a usable `module:` block, write `- module: none` and `- callable: none` and explain the gap in `## Notes`; this is a normal, valid outcome, not an error. + - `## Seams` is required but may contain only `- none` when no runtime boundary needs patching. Otherwise each bullet must be `dotted.import.path = ` and must make the behavioral test deterministic without changing the behavior under test. + - `## Oracle` and `## Negative Cases` bullets are each a single executable Python boolean expression evaluated with `result` bound to the Entry Point's return value — e.g. `result == 200`, `isinstance(result, dict)`, `"error" not in result`, `result.get("status") == "ok"`. Never write prose (no "returned value shape", no "error type" as free text) and never call anything other than a plain method/attribute on `result` or one of `len`, `isinstance`, `str`, `int`, `float`, `bool`, `abs`, `round`, `sorted`, `min`, `max`, `sum`, `any`, `all`, `repr`, `type` — no other function calls, no dunder names, no imports; those bullets are executed, not just displayed. If `## Entry Point` is `- module: none`, or a Story detail genuinely cannot be expressed as such a boolean expression over `result`, omit it from `## Oracle`/`## Negative Cases` and note it under `## Notes` instead. `## Oracle` lists what decides pass/fail; `## Non-Oracle` lists what must NOT matter (prose is fine there — it is not compiled). + - `## Negative Cases` lists forbidden outcomes (from MUST NOT / SHALL NOT rules or obviously-wrong behavior), each as the same kind of boolean expression over `result` (e.g. `result.get("called_llm") is False`). - `## Non-Goals` reflects the issue's own non-goals when stated. @@ -78,16 +90,25 @@ 1. Given , when , then . 2. ... + ## Entry Point + - module: + - callable: + - args: [] + - kwargs: {: } + + ## Seams + - none + ## Oracle - These details matter for pass/fail: - - + These details matter for pass/fail (each bullet a boolean expression over `result`): + - ## Non-Oracle These details should not matter: - ## Negative Cases - - + - ## Non-Goals - diff --git a/pdd/prompts/user_story_tests_python.prompt b/pdd/prompts/user_story_tests_python.prompt index 5218bfbc4..aaacc6a1d 100644 --- a/pdd/prompts/user_story_tests_python.prompt +++ b/pdd/prompts/user_story_tests_python.prompt @@ -66,7 +66,7 @@ precise: - The machine-checkable contract is GENERATED from the human Story + the original issue and lives at `user_stories/contracts/.contract.md`. It carries the top-level (`##`) sections `## Covers`, `## Context`, `## Acceptance Criteria`, - `## Oracle`, `## Non-Oracle`, `## Negative Cases`, `## Non-Goals`, + `## Entry Point`, `## Seams`, `## Oracle`, `## Non-Oracle`, `## Negative Cases`, `## Non-Goals`, `## Candidate Prompts`, `## Notes` (`_REQUIRED_CONTRACT_SECTIONS`), under a `` header used for sync. @@ -147,7 +147,8 @@ varies during normal development goes in `## Non-Oracle`. such as ``, ``, or `` (strip a wrapping ``` ``` ``` fence first). The contract generator (`_llm_generate_story_contract`) accepts only when its output carries the full `_REQUIRED_CONTRACT_SECTIONS` set - (`## Covers`, `## Context`, `## Acceptance Criteria`, `## Oracle`, + (`## Covers`, `## Context`, `## Acceptance Criteria`, `## Entry Point`, + `## Seams`, `## Oracle`, `## Non-Oracle`, `## Negative Cases`, `## Non-Goals`, `## Candidate Prompts`, `## Notes`) and no placeholders. When the LLM is unavailable (no provider key, offline, error) OR the output is empty, missing a required section, or @@ -315,9 +316,29 @@ R2 (MUST NOT): Print the diagnostic block when `quiet` is true. - `generate_user_story` writes the human Story file via `_llm_generate_story_markdown` (requires `## Story`) and then the contract file via `_generate_and_write_contract` / `_llm_generate_story_contract` - (requires the `_REQUIRED_CONTRACT_SECTIONS` set incl. `## Candidate Prompts`). + (requires the `_REQUIRED_CONTRACT_SECTIONS` set incl. `## Entry Point`, + `## Seams`, and `## Candidate Prompts`). It returns failure without writing anything when the human story is unavailable/invalid; a contract failure is non-blocking. + - `## Entry Point`/`## Seams` are populated from `PRIMARY_PROMPT_INTERFACES`, + which `_resolve_prompt_interfaces` builds from each linked prompt's own + `` block PLUS a module path PDD's file-layout convention + derives deterministically from the prompt's path + (`_module_path_for_prompt`) — never from the model. Only a `type: module` + interface with a resolvable module path is offered as usable; a `type: cli` + interface (or one whose path can't be resolved) is listed as explicitly + unsupported so the model cannot invent an import binding for it. + `_validate_contract_entry_point_and_assertions` then rejects (contract not + written; generation fails closed, same as a missing required section) any + LLM output whose `## Entry Point` module/callable isn't exactly one of + those offered pairs (`- module: none` / `- callable: none` is the only + accepted "no deterministic entry point" case), or whose `## Oracle`/ + `## Negative Cases` bullets aren't syntactically-safe boolean expressions + over `result` per `story_test_generator._assertion_from_bullet`'s + allowlist (comparisons/boolean logic/literals/attribute+index access on + `result`, calls only to a small pure-builtin allowlist, no dunder names — + the deepest enforcement point, since these bullets are spliced into + `assert {expr}` in generated pytest that CI executes). - `_llm_generate_story_markdown` takes the resolved issue text/ref (NOT prompt content), loads the `generate_user_story_LLM` meta-prompt (`load_prompt_template`), escapes it via `preprocess(double_curly_brackets=True, diff --git a/pdd/story_test_generation.py b/pdd/story_test_generation.py index de6fc4799..9dc9dcfec 100644 --- a/pdd/story_test_generation.py +++ b/pdd/story_test_generation.py @@ -110,6 +110,22 @@ def _bullets(text: str) -> list[str]: return rows +def _entry_point_has_binding(md_sections: dict[str, str]) -> bool: + """True when ``## Entry Point`` declares a real module/callable pair. + + ``- module: none`` is the contract generator's explicit "no + deterministically-bound callable for this Story" marker (see + ``user_story_tests._validate_contract_entry_point_and_assertions``); such a + contract must fall through to the text-pinning generator below, same as a + contract with no ``## Entry Point`` heading at all. + """ + entry_text = _section(md_sections, "Entry Point").lower() + for bullet in _bullets(entry_text): + if bullet.startswith("module:") and bullet.split(":", 1)[1].strip() == "none": + return False + return True + + def _literal_list(values: list[str], *, indent: str = " ") -> str: if not values: return "[]" @@ -280,8 +296,13 @@ def generate_story_regression_test( # (missing `- module:`/`- callable:`) instead of silently degrading to a # text-pin that the user would mistake for a real behavioral oracle # (pdd#1889 C-F7). A contract with no ## Entry Point at all still falls - # through to the text-pinning generator below. - if "entry point" in md_sections: + # through to the text-pinning generator below. The one deliberate + # exception is `- module: none`: the contract generator's own validation + # (`_validate_contract_entry_point_and_assertions`) only ever writes that + # exact marker when it found no deterministically-bound callable for this + # Story, so it is a confirmed "no entry point" outcome, not a malformed + # one, and also falls through to the text-pinning generator. + if "entry point" in md_sections and _entry_point_has_binding(md_sections): return _generate_behavioral_test(story_path, output) oracle_text = _section(md_sections, "Oracle", "Acceptance Criteria", "Story") diff --git a/pdd/story_test_generator.py b/pdd/story_test_generator.py index f86c0e758..f4b2f0bd8 100644 --- a/pdd/story_test_generator.py +++ b/pdd/story_test_generator.py @@ -81,6 +81,110 @@ def _key_values(section: str) -> dict[str, str]: return values +# Assertion bullets are issue/LLM-derived text that gets spliced verbatim into +# `assert {expr}` in a generated pytest file the story-regression CI lane +# executes -- and that file also has `importlib` and `module` in module scope +# (see `render_story_test`), so a permissive allowlist doesn't just risk +# builtins like `__import__('os').system(...)`; a bare name check alone lets +# `importlib.import_module("os").system(...)` and `module.os.system(...)` +# through too. Syntax validity alone does not make an expression safe, so this +# allowlist constrains bullets to a read-only "compare/inspect `result`" DSL: +# - only `result` and a small set of pure builtins may appear as bare names +# - every attribute/subscript chain must be rooted at `result` (not at +# `module`, `importlib`, or any other name in the generated test's scope) +# - a call is either one of the safe builtins, or a read-only method looked up +# on `result` (never a mutator like `result.clear()`) +_SAFE_ASSERTION_CALL_NAMES = frozenset( + { + "len", "isinstance", "str", "int", "float", "bool", "abs", "round", + "sorted", "min", "max", "sum", "any", "all", "repr", "type", + "dict", "list", "tuple", "set", + } +) +_SAFE_RESULT_METHOD_NAMES = frozenset( + { + "get", "keys", "values", "items", "count", "index", + "lower", "upper", "strip", "lstrip", "rstrip", + "startswith", "endswith", "split", "splitlines", "join", + "isdigit", "isalpha", "isalnum", "isupper", "islower", "isspace", + "copy", + } +) +_SAFE_ASSERTION_NAMES = frozenset({"result"}) | _SAFE_ASSERTION_CALL_NAMES +_SAFE_ASSERTION_NODE_TYPES = ( + ast.Expression, ast.BoolOp, ast.UnaryOp, ast.BinOp, ast.Compare, ast.Call, + ast.Name, ast.Load, ast.Constant, ast.Attribute, ast.Subscript, ast.Slice, + ast.List, ast.Tuple, ast.Dict, ast.Set, + ast.And, ast.Or, ast.Not, ast.Invert, ast.UAdd, ast.USub, + ast.Add, ast.Sub, ast.Mult, ast.Div, ast.FloorDiv, ast.Mod, ast.Pow, + ast.Eq, ast.NotEq, ast.Lt, ast.LtE, ast.Gt, ast.GtE, ast.Is, ast.IsNot, + ast.In, ast.NotIn, +) + + +def _chain_root(node: ast.AST) -> ast.AST: + """Walk down an Attribute/Subscript chain to the value it's rooted at.""" + while isinstance(node, (ast.Attribute, ast.Subscript)): + node = node.value + return node + + +def _is_rooted_at_result(node: ast.AST) -> bool: + root = _chain_root(node) + return isinstance(root, ast.Name) and root.id == "result" + + +def _ensure_safe_assertion(tree: ast.AST, bullet: str) -> None: + """Reject any assertion expression outside the safe result-inspection DSL.""" + for node in ast.walk(tree): + if not isinstance(node, _SAFE_ASSERTION_NODE_TYPES): + raise ValueError( + "Story assertion bullets may only compare/inspect `result` " + f"(unsupported construct {type(node).__name__}): {bullet!r}" + ) + if isinstance(node, ast.Name) and node.id not in _SAFE_ASSERTION_NAMES: + raise ValueError( + "Story assertion bullet references a name other than `result` " + f"or a safe helper: {bullet!r}" + ) + if isinstance(node, ast.Attribute): + if node.attr.startswith("_"): + raise ValueError( + f"Story assertion bullet accesses a private/dunder attribute: {bullet!r}" + ) + if not _is_rooted_at_result(node): + raise ValueError( + "Story assertion bullet accesses an attribute not rooted at " + f"`result`: {bullet!r}" + ) + if isinstance(node, ast.Subscript) and not _is_rooted_at_result(node): + raise ValueError( + f"Story assertion bullet subscripts something other than `result`: {bullet!r}" + ) + if isinstance(node, ast.Call): + func = node.func + if isinstance(func, ast.Name): + if func.id not in _SAFE_ASSERTION_CALL_NAMES: + raise ValueError( + f"Story assertion bullet calls disallowed function {func.id!r}; " + f"only {sorted(_SAFE_ASSERTION_CALL_NAMES)} are permitted: {bullet!r}" + ) + elif isinstance(func, ast.Attribute): + if not _is_rooted_at_result(func): + raise ValueError( + "Story assertion bullet calls a method not rooted at " + f"`result`: {bullet!r}" + ) + if func.attr not in _SAFE_RESULT_METHOD_NAMES: + raise ValueError( + f"Story assertion bullet calls disallowed `result` method " + f"{func.attr!r}; only read-only methods " + f"{sorted(_SAFE_RESULT_METHOD_NAMES)} are permitted: {bullet!r}" + ) + else: + raise ValueError(f"Story assertion bullet has an unsupported call target: {bullet!r}") + + def _assertion_from_bullet(bullet: str) -> str: text = bullet.strip() if text.startswith("assert "): @@ -90,12 +194,13 @@ def _assertion_from_bullet(bullet: str) -> str: if not expr: raise ValueError("Story assertion bullet is empty.") try: - ast.parse(expr, mode="eval") + tree = ast.parse(expr, mode="eval") except SyntaxError as exc: raise ValueError( "Story test generation requires Oracle/Negative Cases bullets to be " - f"Python assertion expressions; got: {bullet!r}" + f"Python assertion expressions over `result`; got: {bullet!r}" ) from exc + _ensure_safe_assertion(tree, bullet) return expr diff --git a/pdd/sync_core/verification.py b/pdd/sync_core/verification.py index cf81d1dc4..c2e4b3ecf 100644 --- a/pdd/sync_core/verification.py +++ b/pdd/sync_core/verification.py @@ -230,6 +230,7 @@ _CONFORMANCE_SPLIT_PROFILE_BYTES[1], _CONFORMANCE_SPLIT_PROFILE_BYTES[1], ) + _PR2316_STALE_LLM_REISSUE_HISTORY_PROFILE_BYTES = ( _OPUS_FABLE_COMPOSED_PROFILE_BYTES[1], _TEMPERATURE_REGRESSION_PROFILE_BYTES[1], diff --git a/pdd/user_story_tests.py b/pdd/user_story_tests.py index 2eff227eb..6a291a5ee 100644 --- a/pdd/user_story_tests.py +++ b/pdd/user_story_tests.py @@ -3,7 +3,9 @@ # pylint: disable=too-many-lines from __future__ import annotations +import ast import hashlib +import inspect import json import logging import os @@ -875,6 +877,8 @@ def resolve_issue_source( # pylint: disable=too-many-return-statements "## Covers", "## Context", "## Acceptance Criteria", + "## Entry Point", + "## Seams", "## Oracle", "## Non-Oracle", "## Negative Cases", @@ -1121,6 +1125,192 @@ def _prompt_inventory_descriptor(prompt_path: Path) -> str: return snippet[:160] +_PDD_INTERFACE_TAG_RE = re.compile( + r"\s*(.*?)\s*", re.DOTALL | re.IGNORECASE +) + + +def _dotted_module_for_code_path(code_path: Path, source_root: Path) -> Optional[str]: + """Return the importable dotted module name for a generated source file.""" + try: + relative = code_path.resolve().relative_to(source_root.resolve()) + except (OSError, RuntimeError, ValueError): + return None + if relative.suffix != ".py": + return None + parts = list(relative.with_suffix("").parts) + if not parts or any(not part.isidentifier() for part in parts): + return None + return ".".join(parts) + + +def _module_path_for_prompt(prompt_path: Path, prompts_root: Optional[Path]) -> Optional[str]: + """Deterministically derive the importable module a prompt compiles to. + + Uses PDD's *actual* prompt -> code mapping (``_prompt_to_code_path`` / + ``_resolve_src_dir``, which honors the ``PDD_SRC_DIR`` override), not a + reinvented layout convention: an Entry Point naming a module PDD itself + wouldn't generate the callable into is worse than no Entry Point at all. + Also requires the mapped source file to actually exist, so this never + offers a module for a prompt whose code hasn't been generated yet. + """ + if prompts_root is None: + return None + code_path = _prompt_to_code_path(prompt_path, prompts_root) + if code_path is None or not code_path.is_file(): + return None + source_root = _resolve_src_dir(prompts_root) + return _dotted_module_for_code_path(code_path, source_root) + + +def _resolve_prompt_interfaces( + prompt_paths: Iterable[Path], prompts_root: Optional[Path] = None +) -> List[Dict[str, object]]: + """Parse each linked prompt's declared ```` and, for a plain + importable module interface, pair it with the deterministic module path + PDD's own convention derives for it (see ``_module_path_for_prompt``). + + Each entry is ``{"name", "module", "functions"}``; ``module``/``functions`` + are empty unless the interface is ``type: module`` AND the module path + could be derived — those are the only entries safe to offer as a + behavioral test Entry Point. + """ + resolved: List[Dict[str, object]] = [] + for prompt_path in _dedupe_prompt_paths(prompt_paths): + try: + text = prompt_path.read_text(encoding="utf-8") + except OSError: + continue + match = _PDD_INTERFACE_TAG_RE.search(text) + if not match: + continue + try: + interface = json.loads(match.group(1)) + except (ValueError, TypeError): + continue + module_path = _module_path_for_prompt(prompt_path, prompts_root) + functions: List[str] = [] + signatures: List[Dict[str, object]] = [] + if isinstance(interface, dict) and interface.get("type") == "module" and module_path: + module_block = interface.get("module") + if isinstance(module_block, dict): + signatures = [ + fn for fn in module_block.get("functions") or [] if isinstance(fn, dict) and fn.get("name") + ] + functions = [fn["name"] for fn in signatures] + resolved.append( + { + "name": prompt_path.name, + "module": module_path if functions else None, + "signatures": signatures, + "functions": functions, + } + ) + return resolved + + +def _format_prompt_interfaces(interfaces: List[Dict[str, object]]) -> str: + """Render resolved interfaces as the ``PRIMARY_PROMPT_INTERFACES`` block.""" + blocks: List[str] = [] + for item in interfaces: + heading = f"### {item['name']}" + module = item.get("module") + functions = item.get("functions") or [] + if module and functions: + sig_lines = "\n".join( + f"- {fn.get('name')}{fn.get('signature', '()')}" + for fn in (item.get("signatures") or []) + if isinstance(fn, dict) and fn.get("name") + ) + blocks.append(f"{heading}\nmodule: {module}\nfunctions:\n{sig_lines}") + else: + blocks.append( + f"{heading}\n(no usable behavioral Entry Point here — interface is " + "not a plain importable module, or its module path could not be " + "determined; do not invent a module or callable for this prompt)" + ) + return "\n\n".join(blocks) or "(no declared pdd-interface found)" + + +def _allowed_entry_points(interfaces: List[Dict[str, object]]) -> Dict[str, Dict[str, Optional[str]]]: + """Map each offered module path to ``{callable_name: declared_signature}``. + + ``declared_signature`` is ``None`` when the interface didn't declare one + for that callable -- callers must treat that as "binding can't be + verified", not "any call binds" (see ``_signature_accepts_call``). + """ + allowed: Dict[str, Dict[str, Optional[str]]] = {} + for item in interfaces: + module = item.get("module") + functions = item.get("functions") or [] + signatures = item.get("signatures") or [] + if not (module and functions): + continue + sig_by_name: Dict[str, Optional[str]] = {} + for fn in signatures: + name = fn.get("name") if isinstance(fn, dict) else None + if isinstance(name, str): + signature = fn.get("signature") + sig_by_name[name] = signature if isinstance(signature, str) else None + allowed[module] = sig_by_name + return allowed + + +def _signature_accepts_call( + signature: str, args: List[object], kwargs: Dict[str, object] +) -> Optional[bool]: + """Return whether calling the declared *signature* with *args*/*kwargs* + would actually bind, using a real ``inspect.Signature.bind`` -- checking + that args/kwargs are Python literals proves nothing about whether they + satisfy the callable's required parameters (review #2397 P1 correctness). + + Returns ``None`` when *signature* can't be parsed into parameters; callers + must treat that as "binding can't be verified", not "binds fine". + """ + from .architecture_sync import ( # pylint: disable=import-outside-toplevel + _parse_signature_parameters, + ) + + parsed = _parse_signature_parameters(signature) + if parsed is None: + return None + kind_map = { + "posonly": inspect.Parameter.POSITIONAL_ONLY, + "arg": inspect.Parameter.POSITIONAL_OR_KEYWORD, + "vararg": inspect.Parameter.VAR_POSITIONAL, + "kwonly": inspect.Parameter.KEYWORD_ONLY, + "kwarg": inspect.Parameter.VAR_KEYWORD, + } + parameters = [] + for param in parsed["parameters"]: + kind = kind_map.get(param["kind"]) + if kind is None: + return None + # The binding check only needs to know whether a parameter CAN be + # omitted, not its actual default value, so any non-empty sentinel + # works here even though the real default may be a non-literal + # expression (e.g. a module-level constant). + default = None if " = " in param["text"] else inspect.Parameter.empty + try: + parameters.append(inspect.Parameter(param["name"], kind, default=default)) + except ValueError: + return None + try: + inspect.Signature(parameters).bind(*args, **kwargs) + except TypeError: + return False + except ValueError: + return None + return True + + +def _primary_prompt_interfaces( + prompt_paths: Iterable[Path], prompts_root: Optional[Path] = None +) -> str: + """Return declared PDD interfaces for linked prompts, when available.""" + return _format_prompt_interfaces(_resolve_prompt_interfaces(prompt_paths, prompts_root)) + + def _scan_prompt_inventory( prompts_dir: Optional[Path], *, @@ -1166,6 +1356,88 @@ def _scan_prompt_inventory( return inventory[:limit] +def _validate_contract_entry_point_and_assertions( + markdown: str, *, allowed_entry_points: Dict[str, Dict[str, Optional[str]]] +) -> Optional[str]: + """Reject a generated contract whose Entry Point or Oracle/Negative Cases + bullets cannot safely compile into a behavioral test. + + Returns an error string, or ``None`` when the contract is valid. The + ``## Entry Point`` ``module: none`` / ``callable: none`` pair is the only + accepted escape hatch for a Story with no deterministically-bound + callable; any other module/callable must exactly match one offered in + ``primary_prompt_interfaces`` (never an invented path), and every + Oracle/Negative Cases bullet must compile as a safe assertion expression + (``story_test_generator._assertion_from_bullet``). + """ + from .story_test_generator import ( # pylint: disable=import-outside-toplevel + _assertion_from_bullet, + _bullet_lines, + _key_values, + _literal_source, + _parse_seams, + _sections, + ) + + sections = _sections(markdown) + entry = _key_values(sections.get("entry point", "")) + module = (entry.get("module") or "").strip() + callable_name = (entry.get("callable") or entry.get("function") or "").strip() + is_none_entry_point = module.lower() == "none" and callable_name.lower() == "none" + if not is_none_entry_point: + if module not in allowed_entry_points: + return ( + f"Entry Point module {module!r} was not offered in " + "primary_prompt_interfaces; invented module paths are rejected." + ) + if callable_name not in allowed_entry_points[module]: + return ( + f"Entry Point callable {callable_name!r} is not declared for module " + f"{module!r}; invented callables are rejected." + ) + try: + args_src = _literal_source(entry.get("args", "[]"), fallback="[]") + kwargs_src = _literal_source(entry.get("kwargs", "{}"), fallback="{}") + args_value = ast.literal_eval(args_src) + kwargs_value = ast.literal_eval(kwargs_src) + if not isinstance(args_value, list): + return "Entry Point args must be a Python list literal." + if not isinstance(kwargs_value, dict): + return "Entry Point kwargs must be a Python dict literal." + except ValueError as exc: + return str(exc) + + signature = allowed_entry_points[module].get(callable_name) + if signature is None: + return ( + f"Entry Point callable {callable_name!r} on module {module!r} has " + "no declared signature to validate args/kwargs against; binding " + "cannot be verified." + ) + binds = _signature_accepts_call(signature, args_value, kwargs_value) + if binds is not True: + return ( + f"Entry Point args={args_value!r} kwargs={kwargs_value!r} do not " + f"bind to {callable_name}{signature}; invented/guessed argument " + "values are rejected." + ) + + seams_text = sections.get("seams", "").strip().lower() + if seams_text not in ("- none", "none"): + try: + _parse_seams(sections.get("seams", "")) + except ValueError as exc: + return str(exc) + + bullets = _bullet_lines(sections.get("oracle", "")) + _bullet_lines(sections.get("negative cases", "")) + for bullet in bullets: + try: + _assertion_from_bullet(bullet) + except ValueError as exc: + return str(exc) + return None + + def _llm_generate_story_contract( # pylint: disable=too-many-arguments,too-many-locals,broad-exception-caught,import-outside-toplevel *, title: str, @@ -1173,6 +1445,8 @@ def _llm_generate_story_contract( # pylint: disable=too-many-arguments,too-many issue_text: str, inventory: List[Tuple[str, str]], primary_refs: List[str], + primary_interfaces: str, + allowed_entry_points: Dict[str, Dict[str, Optional[str]]], strength: float, temperature: float, time: float, @@ -1214,6 +1488,7 @@ def _llm_generate_story_contract( # pylint: disable=too-many-arguments,too-many "ISSUE_TEXT", "PROMPT_INVENTORY", "PRIMARY_PROMPTS", + "PRIMARY_PROMPT_INTERFACES", ], ) try: @@ -1225,6 +1500,7 @@ def _llm_generate_story_contract( # pylint: disable=too-many-arguments,too-many "ISSUE_TEXT": issue_text, "PROMPT_INVENTORY": inventory_block, "PRIMARY_PROMPTS": primary_block, + "PRIMARY_PROMPT_INTERFACES": primary_interfaces, }, strength=strength, temperature=temperature, @@ -1248,6 +1524,12 @@ def _llm_generate_story_contract( # pylint: disable=too-many-arguments,too-many if _contains_placeholder_tokens(markdown): logger.debug("LLM contract contains placeholder tokens.") return None, cost, model + content_error = _validate_contract_entry_point_and_assertions( + markdown, allowed_entry_points=allowed_entry_points + ) + if content_error: + logger.debug("LLM contract failed content validation: %s", content_error) + return None, cost, model if not markdown.endswith("\n"): markdown += "\n" return markdown, cost, model @@ -1294,13 +1576,26 @@ def _generate_and_write_contract( # pylint: disable=too-many-arguments,too-many Returns ``(contract_path, cost, model, error)``. ``contract_path`` is None and ``error`` is set when contract generation could not be completed. """ + extra_prompt_paths = list(extra_prompt_paths) inventory = _scan_prompt_inventory(prompts_root, extra_paths=extra_prompt_paths) + prompt_paths = list(extra_prompt_paths) + if prompts_root is not None: + prompt_paths.extend( + _resolve_prompt_refs_to_paths( + primary_refs, + discover_prompt_files(str(prompts_root), include_llm=True), + prompts_root, + ) + ) + resolved_interfaces = _resolve_prompt_interfaces(prompt_paths, prompts_root) body, cost, model = _llm_generate_story_contract( title=title, story_text=story_text, issue_text=issue_text, inventory=inventory, primary_refs=primary_refs, + primary_interfaces=_format_prompt_interfaces(resolved_interfaces), + allowed_entry_points=_allowed_entry_points(resolved_interfaces), strength=strength, temperature=temperature, time=time, diff --git a/tests/test_story_test_generator.py b/tests/test_story_test_generator.py index cac910a1a..68f5dd4f4 100644 --- a/tests/test_story_test_generator.py +++ b/tests/test_story_test_generator.py @@ -212,3 +212,66 @@ def test_from_story_requires_machine_readable_entrypoint(tmp_path: Path): contract.write_text("## Oracle\n\n- result is not None\n", encoding="utf-8") with pytest.raises(ValueError, match="Entry Point"): generate_story_test(story, tmp_path / "tests" / "test_story.py") + + +# --- Safe-assertion allowlist (review #2397 P1 security) ------------------- +# +# Oracle/Negative Cases bullets are issue/LLM-derived text spliced verbatim +# into `assert {expr}` in a generated pytest file the story-regression CI +# lane executes. `_assertion_from_bullet` previously only checked syntax +# validity, so `__import__('os').system(...)` compiled fine. These tests pin +# the allowlist that now rejects it, at the deepest point the bytes actually +# become executable code -- independent of whatever validation ran upstream +# when the contract was generated. + + +@pytest.mark.parametrize( + "malicious", + [ + "__import__('os').system('touch /tmp/pwned')", + "result.__class__.__mro__[1].__subclasses__()", + "eval('1')", + "(lambda: 1)()", + "__builtins__", + "getattr(result, '__class__')", + ], +) +def test_assertion_from_bullet_rejects_unsafe_expressions(malicious): + from pdd.story_test_generator import _assertion_from_bullet + + with pytest.raises(ValueError): + _assertion_from_bullet(malicious) + + +@pytest.mark.parametrize( + "safe", + [ + 'result == "ok"', + 'result.get("status") == "ok"', + "len(result) == 3", + "isinstance(result, dict)", + '"error" not in result', + 'result.startswith("ok")', + "result[0] == 1 and result[1] == 2", + ], +) +def test_assertion_from_bullet_accepts_safe_expressions(safe): + from pdd.story_test_generator import _assertion_from_bullet + + assert _assertion_from_bullet(safe) == safe + + +def test_generate_story_test_rejects_unsafe_oracle_at_generation_time(tmp_path: Path): + """A malicious Oracle expression must fail `generate_story_test` itself -- + the compiler is the last line of defense even if a contract with unsafe + content somehow reached disk.""" + story = _write_story_contract(tmp_path) + contract = tmp_path / "user_stories" / "contracts" / "checkout_total.contract.md" + contract.write_text( + "## Entry Point\n\n- module: checkout_app\n- callable: checkout_total\n" + "- args: [1, 2]\n- kwargs: {}\n\n" + "## Oracle\n\n- __import__('os').system('true')\n", + encoding="utf-8", + ) + with pytest.raises(ValueError): + generate_story_test(story, tmp_path / "tests" / "test_story.py") diff --git a/tests/test_sync_core_pdd_rollout_policy.py b/tests/test_sync_core_pdd_rollout_policy.py index 935f743bf..848eca82c 100644 --- a/tests/test_sync_core_pdd_rollout_policy.py +++ b/tests/test_sync_core_pdd_rollout_policy.py @@ -2313,6 +2313,11 @@ def test_sync_rollout_repair_executes_the_actual_protected_transition() -> None: verification._SYNC_ROLLOUT_REPAIR_PROFILE_BYTES[0], # pylint: disable=protected-access verification._CONFORMANCE_SPLIT_PROFILE_BYTES[1], # pylint: disable=protected-access ), + # #2397 review: the self-authorizing story-contract rotation + # (_STORY_CONTRACT_ROTATION_POLICY_BYTES / _STORY_CONTRACT_PROFILE_BYTES) + # was removed -- installing and consuming a managed-prompt transition + # in the same PR is forbidden by docs/ci.md. No replacement pair is + # accepted here until a real Phase A rotation is merged to main first. } assert ( hashlib.sha256(_git_blob(SYNC_ROLLOUT_PROTECTED_BASE, ROTATION_FILE)).hexdigest(), diff --git a/tests/test_user_story_tests.py b/tests/test_user_story_tests.py index 21fdd35d3..b7b04200a 100644 --- a/tests/test_user_story_tests.py +++ b/tests/test_user_story_tests.py @@ -2,6 +2,9 @@ # pylint: disable=use-implicit-booleaness-not-comparison,unused-variable # pylint: disable=too-many-locals,line-too-long,too-many-lines +import os +import subprocess +import sys from pathlib import Path from types import SimpleNamespace from unittest.mock import patch @@ -10,6 +13,9 @@ from pdd.user_story_tests import ( _contract_path_for_story, + _generate_and_write_contract, + _module_path_for_prompt, + _primary_prompt_interfaces, _story_content_hash, cache_story_prompt_links, discover_prompt_files, @@ -20,6 +26,7 @@ run_user_story_tests, sync_user_story_contract, ) +from pdd.user_story_tests import _llm_generate_story_contract as _REAL_LLM_GENERATE_STORY_CONTRACT # Issue #1356: stories are authored from the ISSUE, never the prompt. Tests use a @@ -146,6 +153,67 @@ def test_discover_prompt_files_includes_llm(tmp_path): assert {p.name for p in results} == {"foo_python.prompt", "bar_llm.prompt"} +def test_primary_prompt_interfaces_binds_deterministic_module_path(tmp_path): + """A `type: module` interface is paired with the module path PDD's own + file-layout convention derives from the prompt's location -- the model is + handed the exact string, never left to invent one (review #2397 P1).""" + prompts_dir = tmp_path / "app" / "prompts" + prompts_dir.mkdir(parents=True) + prompt = prompts_dir / "checkout_python.prompt" + prompt.write_text( + "\n" + '{"type": "module", "module": {"functions": ' + '[{"name": "checkout_total", "signature": "(items)", "returns": "int"}]}}\n' + "\n", + encoding="utf-8", + ) + + interfaces = _primary_prompt_interfaces([prompt], prompts_dir) + + assert "### checkout_python.prompt" in interfaces + assert "module: app.checkout" in interfaces + assert "checkout_total(items)" in interfaces + + +def test_primary_prompt_interfaces_excludes_cli_interfaces(tmp_path): + """A `type: cli` interface (a Click command) has no direct-call binding -- + it must be listed as explicitly unusable, not silently omitted (which + would let the model believe it's free to invent one) (review #2397 P1).""" + prompts_dir = tmp_path / "app" / "prompts" + prompts_dir.mkdir(parents=True) + prompt = prompts_dir / "context_python.prompt" + prompt.write_text( + "\n" + '{"type": "cli", "cli": {"commands": [{"name": "context", "description": "x"}]}}\n' + "\n", + encoding="utf-8", + ) + + interfaces = _primary_prompt_interfaces([prompt], prompts_dir) + + assert "### context_python.prompt" in interfaces + assert "module:" not in interfaces + assert "no usable behavioral Entry Point" in interfaces + + +def test_primary_prompt_interfaces_without_prompts_root_offers_nothing(tmp_path): + """No `prompts_root` means no module path can be derived -- fail closed + (no Entry Point offered) rather than guess one.""" + prompt = tmp_path / "checkout_python.prompt" + prompt.write_text( + "\n" + '{"type": "module", "module": {"functions": ' + '[{"name": "checkout_total", "signature": "(items)", "returns": "int"}]}}\n' + "\n", + encoding="utf-8", + ) + + interfaces = _primary_prompt_interfaces([prompt]) + + assert "module:" not in interfaces + assert "no usable behavioral Entry Point" in interfaces + + def test_discover_story_files_filters_prefix(tmp_path): stories_dir = tmp_path / "user_stories" stories_dir.mkdir() @@ -818,6 +886,9 @@ def test_generate_user_story_multi_prompt_seeds_cross_module(tmp_path): "## Context\n\n- `prompts/upload_python.prompt`: CSV upload + summary\n\n" "## Acceptance Criteria\n\n" "1. Given a valid CSV, when uploaded, then a summary report is shown.\n\n" + "## Entry Point\n\n" + "- module: upload_app\n- callable: upload_csv\n- args: []\n- kwargs: {}\n\n" + "## Seams\n\n- none\n\n" "## Oracle\n\n- returned value shape\n\n" "## Non-Oracle\n\n- internal helper names\n\n" "## Negative Cases\n\n- Rejecting a valid CSV\n\n" @@ -908,6 +979,9 @@ def test_generate_user_story_multi_prompt_seeds_cross_module(tmp_path): "then the command exits with code 2.\n" "5. Given dynamic tags such as `` or `` are present, when they are " "not expanded, then warning entries are reported without making an LLM call.\n\n" + "## Entry Point\n\n" + "- module: pdd.commands.context\n- callable: context\n- args: []\n- kwargs: {}\n\n" + "## Seams\n\n- none\n\n" "## Oracle\n\n" "These details matter for pass/fail:\n" "- The default output is a usage box with a " @@ -1277,6 +1351,8 @@ def test_generate_writes_two_files_human_story_and_contract(tmp_path): for section in ( "## Covers", "## Acceptance Criteria", + "## Entry Point", + "## Seams", "## Oracle", "## Negative Cases", "## Candidate Prompts", @@ -2311,3 +2387,272 @@ def _no_llm(*_args, **_kwargs): metadata_line = story.read_text(encoding="utf-8").splitlines()[0] assert "demo2_python.prompt" in metadata_line assert "demo_python.prompt" in metadata_line + + +# --- Contract generation -> compiled pytest handoff (review #2397) --------- +# +# The tests above stub `_llm_generate_story_contract` wholesale (see +# `_stub_contract_llm`), so they never exercise its content validation. These +# tests call the REAL function (captured as `_REAL_LLM_GENERATE_STORY_CONTRACT` +# at import time, before the autouse stub patches the module attribute) with +# only the innermost `llm_invoke` call mocked, so they cover the actual +# generation -> validation -> written-contract -> compiled-pytest path. + + +def _write_greeter_app(root: Path, greeting: str = "Hello, {name}!") -> None: + # PDD's actual prompt -> code mapping (`_prompt_to_code_path`): a prompt at + # /greeter_python.prompt compiles to + # /src/greeter.py by default (review #2397 P1 + # correctness/compatibility -- this must be the real default `src/` + # layout, not a reinvented convention). + src_dir = root / "app" / "src" + src_dir.mkdir(parents=True, exist_ok=True) + (src_dir / "greeter.py").write_text( + "def greet(name):\n" + f' return "{greeting}".format(name=name)\n', + encoding="utf-8", + ) + + +def _write_greeter_prompt(root: Path) -> Path: + prompts_dir = root / "app" / "prompts" + prompts_dir.mkdir(parents=True, exist_ok=True) + prompt = prompts_dir / "greeter_python.prompt" + prompt.write_text( + "\n" + '{"type": "module", "module": {"functions": ' + '[{"name": "greet", "signature": "(name)", "returns": "str"}]}}\n' + "\n" + "Greet a user by name.\n", + encoding="utf-8", + ) + return prompt + + +def test_module_path_for_prompt_uses_the_default_src_layout(tmp_path): + """review #2397 P1 correctness/compatibility: the derived module must match + PDD's real prompt -> code mapping (`_prompt_to_code_path`), not a + reinvented convention -- default layout is `/../src/`.""" + prompt = _write_greeter_prompt(tmp_path) + prompts_root = tmp_path / "app" / "prompts" + + assert _module_path_for_prompt(prompt, prompts_root) is None # code not generated yet + + _write_greeter_app(tmp_path) + assert _module_path_for_prompt(prompt, prompts_root) == "greeter" + + +def test_module_path_for_prompt_honors_pdd_src_dir_override(tmp_path, monkeypatch): + """The same mapping must honor `PDD_SRC_DIR`, matching `_resolve_src_dir`.""" + prompt = _write_greeter_prompt(tmp_path) + prompts_root = tmp_path / "app" / "prompts" + custom_src = tmp_path / "custom_src" + custom_src.mkdir() + (custom_src / "greeter.py").write_text( + "def greet(name):\n return f'Hello, {name}!'\n", encoding="utf-8" + ) + monkeypatch.setenv("PDD_SRC_DIR", str(custom_src)) + + assert _module_path_for_prompt(prompt, prompts_root) == "greeter" + + +def _write_greeter_story(root: Path) -> Path: + stories = root / "user_stories" + stories.mkdir(parents=True, exist_ok=True) + story = stories / "story__greeter.md" + story.write_text( + "# User Story: Greeter\n\n" + "## Story\n\n" + "As a user, I get a friendly greeting with my name in it.\n", + encoding="utf-8", + ) + return story + + +def _generate_greeter_contract(tmp_path, contract_markdown): + prompt = _write_greeter_prompt(tmp_path) + # The greeter source must exist before contract generation, matching real + # `pdd generate` usage: only a prompt whose code has actually been + # generated can offer a real Entry Point (review #2397 P1 + # correctness/compatibility). + _write_greeter_app(tmp_path) + story = _write_greeter_story(tmp_path) + story_text = story.read_text(encoding="utf-8") + fake_llm = {"result": contract_markdown, "cost": 0.03, "model_name": "contract-model"} + with ( + patch( + "pdd.user_story_tests._llm_generate_story_contract", + side_effect=_REAL_LLM_GENERATE_STORY_CONTRACT, + ), + patch("pdd.llm_invoke.llm_invoke", return_value=fake_llm), + ): + return story, _generate_and_write_contract( + story_path=story, + story_text=story_text, + title="Greeter", + issue_text="Greet a user by name.", + issue_ref="local", + prompts_root=tmp_path / "app" / "prompts", + extra_prompt_paths=[], + primary_refs=["greeter_python.prompt"], + strength=0.2, + temperature=0.0, + time=0.25, + verbose=False, + ) + + +_GREETER_CONTRACT_MD = ( + "## Covers\n\n- AC1: greets the user by name\n\n" + "## Context\n\n`greet` returns a friendly greeting for a given name.\n\n" + "## Acceptance Criteria\n\n" + "1. Given a name, when greet is called, then it returns a greeting containing that name.\n\n" + "## Entry Point\n\n- module: greeter\n- callable: greet\n- args: [\"World\"]\n- kwargs: {}\n\n" + "## Seams\n\n- none\n\n" + "## Oracle\n\n- result == \"Hello, World!\"\n\n" + "## Non-Oracle\n\n- internal formatting helpers\n\n" + "## Negative Cases\n\n- result != \"Goodbye, World!\"\n\n" + "## Non-Goals\n\n- none\n\n" + "## Candidate Prompts\n\n- none beyond the primary prompt(s)\n\n" + "## Notes\n\n- none\n" +) + + +def test_valid_generated_contract_compiles_to_real_red_green_pytest(tmp_path): + """End-to-end: a validated, LLM-generated contract (module/callable copied + from primary_prompt_interfaces, Oracle/Negative Cases as expressions) is + written, compiles via `generate_story_test` into a real pytest file, and + that file genuinely passes against correct behavior and genuinely fails + once the target regresses -- not a text-pinning tautology.""" + from pdd.story_test_generator import generate_story_test + + story, (contract_path, cost, model, error) = _generate_greeter_contract( + tmp_path, _GREETER_CONTRACT_MD + ) + assert error is None, error + assert contract_path is not None and contract_path.exists() + assert model == "contract-model" + assert cost == pytest.approx(0.03) + + _write_greeter_app(tmp_path, greeting="Hello, {name}!") + output = tmp_path / "tests" / "test_story_greeter.py" + generate_story_test(story, output) + + # The generated test imports the module by its dotted path ("greeter"), + # not by file path, so `app/src` -- where PDD's own convention places the + # generated code -- must be on PYTHONPATH, same as a real project. + run_env = {**os.environ, "PYTHONPATH": str(tmp_path / "app" / "src")} + + passing = subprocess.run( + [sys.executable, "-m", "pytest", str(output), "-q"], + cwd=tmp_path, + env=run_env, + text=True, + capture_output=True, + check=False, + ) + assert passing.returncode == 0, passing.stdout + passing.stderr + + # Seed a real regression in the target behavior -- the SAME generated test + # file must now fail, proving it asserts real behavior, not a tautology. + _write_greeter_app(tmp_path, greeting="Goodbye, {name}!") + failing = subprocess.run( + [sys.executable, "-m", "pytest", str(output), "-q"], + cwd=tmp_path, + env=run_env, + text=True, + capture_output=True, + check=False, + ) + assert failing.returncode != 0 + assert "test_story_greeter" in failing.stdout + + +def test_contract_generation_rejects_invented_entry_point(tmp_path): + """A module/callable not offered in primary_prompt_interfaces is an + invented binding (review #2397 P1) -- generation must fail closed with no + contract written, not silently accept it.""" + bad = _GREETER_CONTRACT_MD.replace( + "- module: greeter\n- callable: greet", + "- module: greeter\n- callable: farewell", + ) + story, (contract_path, _cost, _model, error) = _generate_greeter_contract(tmp_path, bad) + assert contract_path is None + assert error is not None + assert not _contract_path_for_story(story).exists() + + +def test_contract_generation_rejects_args_that_dont_bind_to_the_signature(tmp_path): + """review #2397 P1 correctness: `args: []` for `greet(name)` are literals, + but don't bind -- the emitted test would TypeError before ever reaching + the Oracle. Generation must fail closed instead of writing that contract.""" + bad = _GREETER_CONTRACT_MD.replace( + "- module: greeter\n- callable: greet\n- args: [\"World\"]\n- kwargs: {}", + "- module: greeter\n- callable: greet\n- args: []\n- kwargs: {}", + ) + story, (contract_path, _cost, _model, error) = _generate_greeter_contract(tmp_path, bad) + assert contract_path is None + assert error is not None + assert not _contract_path_for_story(story).exists() + + +def test_contract_generation_rejects_unknown_kwargs(tmp_path): + """An extra kwarg the callable doesn't accept must also fail closed.""" + bad = _GREETER_CONTRACT_MD.replace( + "- module: greeter\n- callable: greet\n- args: [\"World\"]\n- kwargs: {}", + '- module: greeter\n- callable: greet\n- args: ["World"]\n- kwargs: {"loud": true}', + ) + story, (contract_path, _cost, _model, error) = _generate_greeter_contract(tmp_path, bad) + assert contract_path is None + assert error is not None + assert not _contract_path_for_story(story).exists() + + +def test_contract_generation_rejects_prose_oracle_bullets(tmp_path): + """Prose Oracle bullets ("returned value shape") don't compile as Python + assertion expressions (review #2397 P1) -- generation must fail closed.""" + bad = _GREETER_CONTRACT_MD.replace( + '## Oracle\n\n- result == "Hello, World!"\n\n', + "## Oracle\n\n- returned value shape\n\n", + ) + story, (contract_path, _cost, _model, error) = _generate_greeter_contract(tmp_path, bad) + assert contract_path is None + assert error is not None + assert not _contract_path_for_story(story).exists() + + +def test_contract_generation_rejects_unsafe_oracle_expression(tmp_path): + """An Oracle bullet that would execute arbitrary code once spliced into a + generated `assert` (review #2397 P1 security) must be rejected at + generation time, not merely syntax-checked.""" + bad = _GREETER_CONTRACT_MD.replace( + '## Oracle\n\n- result == "Hello, World!"\n\n', + "## Oracle\n\n- __import__('os').system('true')\n\n", + ) + story, (contract_path, _cost, _model, error) = _generate_greeter_contract(tmp_path, bad) + assert contract_path is None + assert error is not None + assert not _contract_path_for_story(story).exists() + + +def test_contract_generation_accepts_explicit_no_entry_point(tmp_path): + """`module: none` / `callable: none` is the deliberate escape hatch for a + Story with no deterministically-bound callable -- it must be accepted, + and the resulting contract must fall back to the text-pinning generator + rather than crash trying to import module "none".""" + from pdd.story_test_generation import generate_story_regression_test + + none_contract = _GREETER_CONTRACT_MD.replace( + "- module: greeter\n- callable: greet\n- args: [\"World\"]\n- kwargs: {}", + "- module: none\n- callable: none\n- args: []\n- kwargs: {}", + ) + story, (contract_path, _cost, _model, error) = _generate_greeter_contract( + tmp_path, none_contract + ) + assert error is None, error + assert contract_path is not None and contract_path.exists() + + result = generate_story_regression_test(story, output=tmp_path / "tests" / "test_story_greeter.py") + assert result.test_count >= 1 + text = result.test_file.read_text(encoding="utf-8") + assert "importlib.import_module" not in text diff --git a/user_stories/contracts/template.contract.md b/user_stories/contracts/template.contract.md index 5af4a0252..14e9b1955 100644 --- a/user_stories/contracts/template.contract.md +++ b/user_stories/contracts/template.contract.md @@ -20,14 +20,26 @@ Describe relevant state, assumptions, fixtures, users, records, external service 1. Given ..., when ..., then ... 2. Given ..., when ..., then ... +## Entry Point + +- module: +- callable: +- args: [] +- kwargs: {} + +## Seams + +Optional runtime-boundary patches for deterministic behavioral tests: +- = + ## Oracle -These details matter for pass/fail: -- error type -- state transition -- absence/presence of external call -- emitted event -- returned value shape +Each bullet is an executable Python boolean expression over `result` (the Entry +Point's return value) — not prose. These decide pass/fail: +- result.status == "ok" +- isinstance(result, dict) +- result.get("event") == "emitted" +- "error" not in result ## Non-Oracle @@ -42,7 +54,8 @@ These details should not matter: ## Negative Cases -List forbidden outcomes this story protects against. +Each bullet is the same kind of boolean expression over `result`, protecting +against a forbidden outcome (e.g. `result.get("called_llm") is False`). ## Non-Goals