From 68aa094623f370c4bb8873db269cf2acf2d5de66 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 12:21:05 -0500 Subject: [PATCH 1/9] Read suites in annotations and unwind sys.modules chains in the OP1 scan Review inject-r4 found two minors that #591 stated in the report instead of fixing. Both fixes land in the scan here. The scan now parses a statement suite an annotation's code call holds, with the same memo, depth limit and unexpanded handling as a suite in a statement. Expression literals in hints stay with the hint checks. module_path now unwinds sys.modules subscripts and attributes in one loop, so a long chain of either cannot exhaust recursion. Co-Authored-By: Claude Opus 5.5 --- .../open_population/test_stream_injection.py | 485 +++++++++++++++--- 1 file changed, 409 insertions(+), 76 deletions(-) diff --git a/tests/open_population/test_stream_injection.py b/tests/open_population/test_stream_injection.py index 9f5872fc..52e9021b 100644 --- a/tests/open_population/test_stream_injection.py +++ b/tests/open_population/test_stream_injection.py @@ -1184,8 +1184,8 @@ def _relocate(tree: ast.AST, origin: ast.AST) -> None: setattr(node, field, getattr(origin, field, None)) -def _annotation_nodes(tree: ast.AST) -> set[int]: - """Return the ids of every node inside an annotation.""" +def _annotations(tree: ast.AST) -> list[ast.expr]: + """List every annotation in a tree.""" hints = [] for node in ast.walk(tree): if isinstance(node, ast.arg | ast.AnnAssign) and node.annotation: @@ -1193,7 +1193,78 @@ def _annotation_nodes(tree: ast.AST) -> set[int]: elif isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): if node.returns is not None: hints.append(node.returns) - return {id(item) for hint in hints for item in ast.walk(hint)} + return hints + + +def _spelled(node: ast.AST, memo: _FoldMemo) -> ast.expr | None: + """Return the expression a node's text spells, if it spells one.""" + text = _static_text(node, memo) if isinstance(node, ast.expr) else None + if text is None: + return None + try: + return ast.parse(text.strip(), mode="eval").body + except (SyntaxError, ValueError): + return None + + +def _expanded( + call: ast.Call, + memo: _FoldMemo, + unexpanded: set[int], + depth: int, + anchor: ast.AST, +) -> list[ast.Module]: + """Parse the literal code a call runs, nested code too. + + The parsed tree takes the call's location. Code nested more than + ``_EMBED_DEPTH`` deep is not parsed: ``anchor``, a node the scan walks, + goes in ``unexpanded``, and the scan reports it as dynamic code. + """ + text = _static_text(call.args[0], memo) + if text is None: + return [] + try: + parsed = ast.parse(text) + except (SyntaxError, ValueError): + return [] + if depth > _EMBED_DEPTH: + unexpanded.add(id(anchor)) + return [] + _relocate(parsed, call) + return [parsed, *_embedded(parsed, memo, unexpanded, depth + 1)] + + +def _hint_code( + part: ast.expr, + hint: ast.expr, + memo: _FoldMemo, + unexpanded: set[int], + depth: int, +) -> list[ast.Module]: + """Parse the statement suites a hint's code calls hold, nested too. + + A literal that spells an expression is the hint checks' to read, so + the walk only follows it, one level deeper, for a suite inside. A code + call's other literal is code, as it is outside annotations. A spelled + part is no node the scan walks, so ``hint`` stands for it, and it goes + in ``unexpanded`` when such a part runs code the folder cannot read. + """ + found = [] + for node in ast.walk(part): + spelled = _spelled(node, memo) + if spelled is not None: + _relocate(spelled, hint) + found += _hint_code(spelled, hint, memo, unexpanded, depth + 1) + elif ( + isinstance(node, ast.Call) + and node.args + and _code_call(node.func) + and _spelled(node.args[0], memo) is None + ): + if part is not hint and _static_text(node.args[0], memo) is None: + unexpanded.add(id(hint)) + found += _expanded(node, memo, unexpanded, depth, hint) + return found def _embedded( @@ -1202,35 +1273,23 @@ def _embedded( """Parse the literal code a module evaluates, nested code too. A literal passed to ``eval``, ``exec``, ``compile`` or ``ForwardRef`` - is code the module runs, so the scan checks it as part of the module; - each parsed tree takes its call's location. Inside an annotation the - hint checks parse such literals instead, so they are skipped here. - Code nested more than ``_EMBED_DEPTH`` deep is not parsed: its call - goes in ``unexpanded``, and the scan reports it as dynamic code. + is code the module runs, so the scan checks it as part of the module. + Inside an annotation the hint checks read literals that spell an + expression, and ``_hint_code`` finds the rest. """ + hints = _annotations(tree) + hinted = {id(item) for hint in hints for item in ast.walk(hint)} found = [] - hinted = _annotation_nodes(tree) for node in ast.walk(tree): - if not ( + if ( isinstance(node, ast.Call) and node.args and _code_call(node.func) and id(node) not in hinted ): - continue - text = _static_text(node.args[0], memo) - if text is None: - continue - try: - parsed = ast.parse(text) - except (SyntaxError, ValueError): - continue - if depth > _EMBED_DEPTH: - unexpanded.add(id(node)) - continue - _relocate(parsed, node) - found.append(parsed) - found += _embedded(parsed, memo, unexpanded, depth + 1) + found += _expanded(node, memo, unexpanded, depth, node) + for hint in hints: + found += _hint_code(hint, hint, memo, unexpanded, depth) return found @@ -1325,32 +1384,41 @@ def _resolve(self, path: str) -> str: return path def module_path(self, node: ast.expr) -> str | None: - """Return the dotted path a name, attribute or import denotes.""" + """Return the dotted path a name, attribute or import denotes. + + Attributes and literal ``sys.modules`` reads are unwound in a loop, + so a long chain cannot exhaust recursion. + """ + links: list[str | ast.expr] = [] + while True: + if isinstance(node, ast.Attribute): + links.append(node.attr) + node = node.value + elif ( + isinstance(node, ast.Subscript) + and isinstance(node.value, ast.Attribute) + and node.value.attr == "modules" + ): + links.append(node.slice) + node = node.value.value + else: + break + path = None if isinstance(node, ast.Name): path = self.modules.get(node.id) - return None if path is None else self._resolve(path) - if isinstance(node, ast.Attribute): - # Unwound in a loop, so a long chain cannot exhaust recursion. - attributes = [] - while isinstance(node, ast.Attribute): - attributes.append(node.attr) - node = node.value - path = self.module_path(node) - for attribute in reversed(attributes): + path = None if path is None else self._resolve(path) + elif isinstance(node, ast.Call): + path = self._imported(node) + for link in reversed(links): + if isinstance(link, str): if not path: return None - path = self._resolve(f"{path}.{attribute}") - return path - if isinstance(node, ast.Call): - return self._imported(node) - if ( - isinstance(node, ast.Subscript) - and isinstance(node.value, ast.Attribute) - and node.value.attr == "modules" - and self.module_path(node.value.value) == "sys" - ): - return self.text(node.slice) - return None + path = self._resolve(f"{path}.{link}") + elif path != "sys": + return None + else: + path = self.text(link) + return path def _imported(self, node: ast.Call) -> str | None: """Name the module a literal ``import_module`` or ``__import__`` gives.""" @@ -1866,12 +1934,8 @@ def _hint_parts(self, hint: ast.expr) -> list[ast.expr]: index = 0 while index < len(parts) and len(parts) < _HINT_PARTS: for node in ast.walk(parts[index]): - text = self.text(node) if isinstance(node, ast.expr) else None - if text is None: - continue - try: - parsed = ast.parse(text.strip(), mode="eval").body - except (SyntaxError, ValueError): + parsed = _spelled(node, self.memo) + if parsed is None: continue _relocate(parsed, hint) for item in ast.walk(parsed): @@ -2061,6 +2125,9 @@ def _benign_instance_use(self, node: ast.expr) -> bool: def _kind(self, node: ast.AST) -> str | None: """Name the specific check ``node`` makes, if it makes one.""" + if id(node) in self.unexpanded: + # Code nested past the depth the scan parses may name anything. + return "dynamic-code" if isinstance(node, ast.expr) and not isinstance( node, ast.Name | ast.Attribute ): @@ -2154,14 +2221,10 @@ def _call_kind(self, node: ast.Call) -> str | None: if ( _code_call(node.func) and node.args - and ( - id(node) in self.unexpanded - or (self.text(node.args[0]) is None and self.binds_tracked()) - ) + and self.text(node.args[0]) is None + and self.binds_tracked() ): - # Code the folder cannot read may name what the module binds, - # and code nested past the depth the scan parses may name - # anything. + # Code the folder cannot read may name what the module binds. return "dynamic-code" if ( name in _INTROSPECTION_CALLS @@ -2405,13 +2468,21 @@ def _inert(node: ast.AST) -> bool: return True +def _is_hint(parent: ast.AST, field: str) -> bool: + """Return whether this child of ``parent`` is an annotation.""" + if isinstance(parent, ast.FunctionDef | ast.AsyncFunctionDef): + return field == "returns" + return isinstance(parent, ast.arg | ast.AnnAssign) and ( + field == "annotation" + ) + + def _folds_through(parent: ast.AST, field: str) -> bool: - """Return whether the folder or a hint parser reads this child. + """Return whether the folder reads this child. It reads operands, subscripts, f-string parts, method receivers, the arguments of the calls it folds and of the calls whose literal is - code, the items of a container it reads, and annotations, whose - strings are parsed as code. + code, and the items of a container it reads. """ if isinstance( parent, @@ -2425,14 +2496,9 @@ def _folds_through(parent: ast.AST, field: str) -> bool: | ast.Tuple | ast.List | ast.Set - | ast.Dict - | ast.arg, + | ast.Dict, ): return True - if isinstance(parent, ast.FunctionDef | ast.AsyncFunctionDef): - return field == "returns" - if isinstance(parent, ast.AnnAssign): - return field == "annotation" if isinstance(parent, ast.Call) and field in {"args", "keywords"}: return ( _call_name(parent.func) @@ -2445,32 +2511,34 @@ def _prune(tree: ast.Module) -> ast.Module: """Replace each inert literal container with ``None``, in place. No check reports or binds anything from an inert container, and none - is pruned where the folder or a hint parser could read it, so the hits - are unchanged (a differential test pins that); the data modules' large - tables stop costing time and memory. + is pruned where the folder could read it or in an annotation, whose + strings are parsed as code, so the hits are unchanged (a differential + test pins that); the data modules' large tables stop costing time and + memory. """ - stack: list[tuple[ast.AST, bool]] = [(tree, False)] + stack: list[tuple[ast.AST, bool, bool]] = [(tree, False, False)] while stack: - node, inherited = stack.pop() + node, inherited, annotated = stack.pop() for field, value in ast.iter_fields(node): items = value if isinstance(value, list) else [value] for index, item in enumerate(items): if not isinstance(item, ast.AST): continue + hinted = annotated or _is_hint(node, field) # A keyword's value is read as its call's arguments are. read = ( inherited if isinstance(node, ast.keyword) else _folds_through(node, field) ) - if not read and _inert(item): + if not (read or hinted) and _inert(item): stub = ast.copy_location(ast.Constant(None), item) if isinstance(value, list): value[index] = stub else: setattr(node, field, stub) else: - stack.append((item, read)) + stack.append((item, read, hinted)) return tree @@ -3903,6 +3971,252 @@ def test_scan_parses_five_levels_of_literal_code_and_flags_deeper(depth): ) +_SUITE = "global f\ndef f(x):\n return isinstance(x, R)\n" +# Each reads an annotation, which runs the code call in it. R is an alias, +# so the hint names nothing the read itself would report. +_FORCED_HINTS = { + "class": "class C:\n v: {}\nC.__annotations__\n", + "argument": "def g(v: {}):\n pass\ng.__annotations__\n", + "return": "def g() -> {}:\n pass\ng.__annotations__\n", +} + + +def _wrapped(code, depth): + """Nest ``code`` in ``depth`` literal execs.""" + for _ in range(depth): + code = f"exec({code!r})" + return code + + +@pytest.mark.parametrize("depth", (1, 5, 6, 7)) +@pytest.mark.parametrize("place", tuple(_FORCED_HINTS)) +def test_scan_reads_statement_suites_an_annotation_runs(place, depth): + """A suite in a hint counts to five levels, as one in a statement does. + + Every outer literal here spells an expression, and only the innermost + is a suite. Each module is a working check once its hint is read, so + the scan reports it as the check, or as dynamic code past five levels. + """ + hint = _FORCED_HINTS[place].format(_wrapped(_SUITE, depth)) + text = f"{_IMPORT} as R\n{hint}" + assert _answers(text) == [True, False] + line = 3 if place == "class" else 2 + kind = "isinstance" if depth <= 5 else "dynamic-code" + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == [ + (line, kind) + ], describe(hits) + + +_RUN_SUITE = _wrapped(_SUITE, 1) +_RUN_DEEP_SUITE = _wrapped(_SUITE, 6) +_ANNOTATED_SUITE = f"class C:\n v: {_RUN_SUITE}\nC.__annotations__\n" +_TRUSTED_HINT = f"def g(v: R, w: {_RUN_SUITE}):\n pass\ng.__annotations__\n" +# Working checks an annotation hides, the hits they give, and whether the +# forcing read runs them. Line 1 imports the registry as R. +HINT_CODE_CASES = { + "a folded literal": ( + f"class C:\n v: exec({_SUITE[:9]!r} + {_SUITE[9:]!r})\n" + "C.__annotations__\n", + [(3, "isinstance")], + ), + "compiled code": ( + f"class C:\n v: exec(compile({_SUITE!r}, 'f', 'exec'))\n" + "C.__annotations__\n", + [(3, "dynamic-code"), (3, "isinstance")], + ), + "an expression literal around the suite": ( + f"class C:\n v: eval({_RUN_SUITE!r})\nC.__annotations__\n", + [(3, "isinstance")], + ), + "a string hint": ( + f"import typing\ndef g(v: {_RUN_SUITE!r}):\n pass\n" + "typing.get_type_hints(g)\n", + [(3, "isinstance")], + ), + "a string hint six levels deep": ( + f"import typing\ndef g(v: {_RUN_DEEP_SUITE!r}):\n pass\n" + "typing.get_type_hints(g)\n", + [(3, "dynamic-code")], + ), + "a string hint running code the folder cannot read": ( + f"import typing\nS = {_SUITE!r}\ndef g(v: 'exec(S)'):\n pass\n" + "typing.get_type_hints(g)\n", + [(4, "dynamic-code")], + ), + "a keyword-only argument": ( + f"def g(*, v: {_RUN_SUITE}):\n pass\ng.__annotations__\n", + [(2, "isinstance")], + ), + "a starred argument": ( + f"def g(*v: {_RUN_SUITE}):\n pass\ng.__annotations__\n", + [(2, "isinstance")], + ), + "a coroutine": ( + f"async def g(v: {_RUN_SUITE}):\n pass\ng.__annotations__\n", + [(2, "isinstance")], + ), + "an annotation in literal code": ( + f"exec({_ANNOTATED_SUITE!r})\n", + [(2, "isinstance")], + ), + "a trusted function's hint": ( + _TRUSTED_HINT, + [(2, "isinstance"), (4, "annotation-read")], + ), + "a trusted function's hint at the sixth level": ( + f"{_wrapped(_TRUSTED_HINT, 5)}\n", + [(2, "dynamic-code")], + ), + "a literal under a call that does not fold": ( + f"class C:\n v: (lambda s=[{_RUN_SUITE!r}]: eval(s[0]))()\n" + "C.__annotations__\n", + [(3, "dynamic-code"), (3, "isinstance")], + ), +} + + +@pytest.mark.parametrize("case", tuple(HINT_CODE_CASES)) +def test_scan_reads_code_an_annotation_hides(case): + """Each case is a working check, and both scans report it alike.""" + body, expected = HINT_CODE_CASES[case] + text = f"{_IMPORT} as R\n{body}" + assert _answers(text) == [True, False] + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == expected, describe( + hits + ) + + +@pytest.mark.parametrize( + "hint", + ( + "exec('pass')", + "exec('x = 1')", + "exec(code)", + "eval('int')", + "'int'", + "eval('exec(\"pass\")')", + ), +) +def test_scan_trusts_code_in_a_hint_that_reaches_no_class(hint): + """A hint's code call alone reports nothing in a module binding none.""" + assert scan_sources([_invented(f"def g(v: {hint}):\n pass\n")]) == () + + +def test_scan_leaves_expression_literals_in_hints_to_the_hint_checks(): + """A trusted hint stays trusted: only a suite counts as code.""" + text = ( + f"{_REGISTRY_IMPORT}" + "from typing import ForwardRef\n" + "def g(v: ForwardRef('ProjectionRNGRegistry')):\n pass\n" + "def h(v: eval('ProjectionRNGRegistry')):\n pass\n" + ) + assert scan_sources([_invented(text)]) == () + + +def test_scan_reads_a_suite_in_a_hint_as_it_reads_one_in_a_statement(): + """An annotation runs a literal suite as an ``exec`` statement does. + + So the hint gives every check the statement gives. The catch-all's + name-string hits do not carry over: it reads the string in a statement + and skips it in a trusted annotation. + """ + + def checks(text): + hits = scan_sources([_invented(text)]) + return { + hit + for hit in hits + if hit.kind not in {"class-name-string", "module-name-string"} + } + + texts = [text for text, _ in SPELLINGS.values()] + texts += [_REGISTRY_IMPORT + body for body in NEGATIVE_BODIES] + for text in texts: + assert checks(f"exec({text!r})\n") <= checks( + f"def g(v: exec({text!r})):\n pass\n" + ) + + +def _recursive_module_path(scope, node): + """Follow a chain by recursion, which only a short chain survives.""" + if isinstance(node, ast.Name): + path = scope.modules.get(node.id) + return None if path is None else scope._resolve(path) + if isinstance(node, ast.Attribute): + path = _recursive_module_path(scope, node.value) + return scope._resolve(f"{path}.{node.attr}") if path else None + if isinstance(node, ast.Call): + return scope._imported(node) + if ( + isinstance(node, ast.Subscript) + and isinstance(node.value, ast.Attribute) + and node.value.attr == "modules" + and _recursive_module_path(scope, node.value.value) == "sys" + ): + return scope.text(node.slice) + return None + + +_PATH_BASES = st.sampled_from( + [ + "sys", + "os", + "engine", + "rng", + "unbound", + "__import__('sys')", + "__import__('os')", + "importlib.import_module('sys')", + "(lambda: sys)()", + "values[0]", + ] +) +_PATH_LINKS = st.lists( + st.sampled_from( + [ + ".modules['sys']", + ".modules['os']", + ".modules['']", + ".modules[key]", + ".modules['populace_dynamics.engine.rng']", + ".modules", + ".path", + ".rng", + ".ProjectionRNGRegistry", + "['sys']", + ".other['sys']", + "()", + ] + ), + max_size=8, +) + + +@settings(max_examples=500, deadline=None) +@given(base=_PATH_BASES, links=_PATH_LINKS) +def test_module_path_agrees_with_a_recursive_walk(base, links): + """Unwinding a chain changes no path, and keeps its determinism.""" + source = _invented( + "import importlib\nimport os\nimport sys\n" + "import populace_dynamics.engine as engine\n" + "from populace_dynamics.engine import rng\n" + ) + scope = _ModuleBindings(source, set(), set(), {}, _FoldMemo(source.path)) + scope.bind( + [ + node + for node in ast.walk(_parse(source)) + if isinstance(node, _BINDING_NODES) + ] + ) + chain = ast.parse(base + "".join(links), mode="eval").body + assert scope.module_path(chain) == _recursive_module_path(scope, chain) + + _CHAIN = 3000 _RUN_ANY = "isinstance(x, ProjectionRNGRegistry) if x else x" # Checks inside, or next to, chains 3000 links long. The recursive walks @@ -3953,6 +4267,22 @@ def test_scan_parses_five_levels_of_literal_code_and_flags_deeper(depth): + "isinstance(x, ProjectionRNGRegistry) # check\n", "isinstance", ), + # A link of these holds two nodes, so half as many reach the same depth. + "a chain of sys.modules reads": ( + f"{_IMPORT}\nimport sys\nSAME = sys" + + ".modules['sys']" * (_CHAIN // 2) + + "\ndef f(x):\n" + " return isinstance(x, ProjectionRNGRegistry) # check\n", + "isinstance", + ), + "a chain of sys.modules reads to the class": ( + "import sys\nimport populace_dynamics.engine.rng\n" + "def f(x):\n return isinstance(x, sys" + + ".modules['sys']" * (_CHAIN // 2) + + ".modules['populace_dynamics.engine.rng'].ProjectionRNGRegistry)" + " # check\n", + "isinstance", + ), } @@ -4017,6 +4347,9 @@ def test_scan_reads_long_chains_without_exhausting_recursion(shape): + " b = ''.join({0}) == type(x).__name__ or ({0},)[0] == x\n" + " c = '%s' % ({0},) == '{{}}'.format({0}) == str({0}).lower()\n" + " return a or b or c or g(key={0})\n" + + "class K:\n" + + " v: g({0})\n" + + " w: (lambda y={0}: y)()\n" ) @@ -4037,7 +4370,7 @@ def test_pruning_leaves_scans_unchanged_for_any_literal(literal): Each generated literal is placed in every position a check could read it from: a binding, a container beside the class, a listing, a - default, a loop, a comparison, a type check and a call. + default, a loop, a comparison, a type check, a call and an annotation. """ text = _LITERAL_PLACES.format(repr(literal)) pruned = scan_sources([_invented(text)]) From bcb89d4c6650d06f0f3855102e6fa2ac8c55fd85 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 12:23:24 -0500 Subject: [PATCH 2/9] Tighten the hint parity test and pin pruning at every hint place Co-Authored-By: Claude Opus 5.5 --- .../open_population/test_stream_injection.py | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/tests/open_population/test_stream_injection.py b/tests/open_population/test_stream_injection.py index 52e9021b..b9b8c17c 100644 --- a/tests/open_population/test_stream_injection.py +++ b/tests/open_population/test_stream_injection.py @@ -4069,11 +4069,6 @@ def test_scan_reads_statement_suites_an_annotation_runs(place, depth): f"{_wrapped(_TRUSTED_HINT, 5)}\n", [(2, "dynamic-code")], ), - "a literal under a call that does not fold": ( - f"class C:\n v: (lambda s=[{_RUN_SUITE!r}]: eval(s[0]))()\n" - "C.__annotations__\n", - [(3, "dynamic-code"), (3, "isinstance")], - ), } @@ -4090,6 +4085,21 @@ def test_scan_reads_code_an_annotation_hides(case): ) +@pytest.mark.parametrize("place", tuple(_FORCED_HINTS)) +def test_pruning_keeps_each_hint_whole(place): + """A literal in a hint can run, though nothing folds its way to it.""" + hint = f"(lambda s=[{_RUN_SUITE!r}]: eval(s[0]))()" + text = f"{_IMPORT} as R\n{_FORCED_HINTS[place].format(hint)}" + assert _answers(text) == [True, False] + line = 3 if place == "class" else 2 + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == [ + (line, "dynamic-code"), + (line, "isinstance"), + ], describe(hits) + + @pytest.mark.parametrize( "hint", ( @@ -4120,9 +4130,10 @@ def test_scan_leaves_expression_literals_in_hints_to_the_hint_checks(): def test_scan_reads_a_suite_in_a_hint_as_it_reads_one_in_a_statement(): """An annotation runs a literal suite as an ``exec`` statement does. - So the hint gives every check the statement gives. The catch-all's - name-string hits do not carry over: it reads the string in a statement - and skips it in a trusted annotation. + So the hint gives every check the statement gives, with two exceptions. + The catch-all reads the string in a statement, where it holds a tracked + name, and skips it in a trusted annotation. And a hint that names a + tracked class makes a namespace read an annotation read. """ def checks(text): @@ -4136,9 +4147,10 @@ def checks(text): texts = [text for text, _ in SPELLINGS.values()] texts += [_REGISTRY_IMPORT + body for body in NEGATIVE_BODIES] for text in texts: - assert checks(f"exec({text!r})\n") <= checks( - f"def g(v: exec({text!r})):\n pass\n" - ) + plain = checks(f"exec({text!r})\n") + hinted = checks(f"def g(v: exec({text!r})):\n pass\n") + added = {hit.kind for hit in hinted - plain} + assert plain <= hinted or added == {"annotation-read"} def _recursive_module_path(scope, node): From 9d81e218cabc6fe29fdb3922e73480faea0bf5e8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 13:08:08 -0500 Subject: [PATCH 3/9] Pin the mutants the first OP1 scan sweep left alive The sweep showed that no test reached _resolve from module_path, the empty path arm or the strip in _spelled. Compare module_path with a recursive reference on every chain of up to three links with invented re-exports, and add working cases for a padded literal and a re-exported sys. Count levels through every mix of exec and eval, and make the second sys.modules chain one whose hit depends on the path. State the new behaviour in section 6. Co-Authored-By: Claude Opus 5.5 --- .../open_population_evidence/op1_report.md | 81 +++++++--- .../open_population/test_stream_injection.py | 147 +++++++++++++----- 2 files changed, 167 insertions(+), 61 deletions(-) diff --git a/docs/design/open_population_evidence/op1_report.md b/docs/design/open_population_evidence/op1_report.md index 950d8194..30172c97 100644 --- a/docs/design/open_population_evidence/op1_report.md +++ b/docs/design/open_population_evidence/op1_report.md @@ -918,11 +918,18 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at code passed to `eval`, `exec`, `compile` or `ForwardRef` is parsed and scanned with the module, and so is literal code nested in it, down to five levels. A call whose literal code would be a sixth level is - reported as `dynamic-code` whatever the module binds. Annotations - differ. There only the hint checks parse literals, and they read - expressions, so a statement suite in an annotation, such as - `v: exec("def f(x): ...")`, stays unparsed at every depth and gives no - hit (limit class 4). + reported as `dynamic-code` whatever the module binds. A code call in + an annotation follows the same rules when its literal does not parse + as an expression, as in `v: exec("def f(x): ...")`, and the levels + count the same way. A literal that does parse as an expression stays + with the hint checks, so `ForwardRef("ProjectionRNGRegistry")` stays + trusted, and the scan looks inside it only for a suite. The scan + never asks whether Python evaluates the annotation, so a local + variable's hint, which Python never runs, gives the same hit. Where a + sixth level, or a code call on text the folder cannot read, sits in a + literal inside an annotation, the scan walks no node that holds it, so + the annotation itself reports `dynamic-code`, whatever the module + binds. - `member-introspection`: a dunder read on, or an introspection call such as `inspect.getmodule` or `getattr` on, a member of a module through which a class can be reached. @@ -1001,21 +1008,30 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at exception it allows is the four-digit width rule; with that rule off, the folder must agree. - **Deep expressions.** The walks that follow class info, instances and - module paths keep their own stacks, as the folder and the catch-all do, - with one exception: `module_path` still recurses through literal - `sys.modules[...]` subscripts. Beside a working check, such a chain - passes at 500 links and raises `RecursionError` at 1000. The error stops - the scan, so it cannot silently miss a check. - `test_scan_reads_long_chains_without_exhausting_recursion` gives seven - working checks inside or beside chains 3000 links long: a sum of - literals, a union, slices, conditionals, attributes, calls and - negations. At `5a4fc14c`, the sum, the union, the slices and the - attributes raised `RecursionError` at 1000 links. + module paths keep their own stacks, as the folder and the catch-all do. + `module_path` unwinds attributes and literal `sys.modules[...]` reads in + one loop. + `test_scan_reads_long_chains_without_exhausting_recursion` gives nine + working checks inside or beside long chains. Seven run 3000 links: a sum + of literals, a union, slices, conditionals, attributes, calls and + negations. Two run 1500 `sys.modules` reads of two nodes each, so they + reach the same depth. One sits beside its check, and the other ends in + the module its check takes the class from. At `5a4fc14c`, the sum, the + union, the slices and the attributes raised `RecursionError` at 1000 + links, and at `6413d842` a `sys.modules` chain did too. + `test_module_path_agrees_with_a_recursive_walk_on_short_chains` compares + `module_path` with a recursive reference on all 26,180 chains of up to + three links, invented re-exports included. + `test_module_path_agrees_with_a_recursive_walk` does the same on 200 + Hypothesis chains of four to eight links, and + `test_scan_reads_sys_modules_through_a_re_exported_sys` reaches a module + through a re-exported `sys`, by name and by attribute. - **Cost.** Right after parsing, each literal container of constants is replaced by `None` unless a string in it holds either class's or module's name in any case, a probe name, or a name a read of a trusted - place spells, or the folder could read it. No check can report or bind - anything from such a container. The generated data modules of + place spells, or the folder could read it, or it sits in an annotation. + No check can report or bind anything from such a container. The + generated data modules of 18.5, 11.1 and 1.6 MB are almost all such containers: pruning leaves 178, 78 and 7 of their 706,994, 251,826 and 85,417 syntax-tree nodes, and 934,121 of the 2,007,909 in all 302 modules at `a747bafa`. At `9cced8ff` @@ -1023,7 +1039,7 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at source changes between the two. `test_pruning_leaves_every_invented_scan_unchanged` and `test_pruning_leaves_scans_unchanged_for_any_literal` (200 Hypothesis - literals, each placed in fourteen positions, folded ones among them) + literals, each placed in sixteen positions, folded ones among them) check that pruning changes no result. On the host, at a one-minute load near 100, the real-tree test took 15.0 s and the census test 4.8 s. A process that parses the tree peaks near @@ -1066,6 +1082,32 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at a working `isinstance` check in one to seven literal `exec` calls. It is reported as `isinstance` down to five levels and as `dynamic-code` at six and seven. + - `test_scan_reads_statement_suites_an_annotation_runs` hides a working + check in class, argument and return annotations, in one, five, six and + seven literal `exec` calls, and reads the annotation to run it. Pruned + and unpruned, the scan reports `isinstance` down to five levels and + `dynamic-code` at six and seven. + `test_scan_counts_levels_through_every_mix_of_code_calls` does the same + for all 127 mixes of `exec` and `eval` in up to six outer calls, at each + of the three places. + - `test_scan_reads_code_an_annotation_hides` gives thirteen more working + checks: a folded literal, compiled code, an expression literal around + the suite, a padded one, three string hints, a keyword-only and a + starred argument, + a coroutine, an annotation inside literal code, and a trusted + function's hint at the first and sixth levels. + `test_pruning_keeps_each_hint_whole` puts a working check's literal + under a lambda's default in each of the three annotation places. + - `test_scan_trusts_code_in_a_hint_that_reaches_no_class` gives six + hints in a module that binds nothing tracked, and + `test_scan_leaves_expression_literals_in_hints_to_the_hint_checks` + keeps `ForwardRef` and `eval` hints that name the registry trusted. + - `test_scan_reads_a_suite_in_a_hint_as_it_reads_one_in_a_statement` + wraps each spelling and negative in a hint and in an `exec` statement. + The hint gives every check the statement gives, with two exceptions. + The catch-all reads the statement's string, so its name hits do not + carry over. And a hint that names a tracked class makes a namespace + read an `annotation-read`. - **Its messages.** A hit carries only a module path, line number and node kind. `test_scan_messages_never_show_source_text` plants a distinctive literal in an invented snippet and checks that it is absent from the @@ -1097,8 +1139,7 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at 3. **Files read as data**, a module's own source among them, as with `open(__file__)` or `importlib.resources`. 4. **Code it does not read**: the excluded modules, OP1's own package, - scripts, tests, third-party code and literal statement suites inside - annotations. + scripts, tests and third-party code. So a clean scan is evidence, not proof. Behaviour is checked separately, at run time: on the paths the reader runs and §4's identity diff --git a/tests/open_population/test_stream_injection.py b/tests/open_population/test_stream_injection.py index b9b8c17c..0c67a782 100644 --- a/tests/open_population/test_stream_injection.py +++ b/tests/open_population/test_stream_injection.py @@ -4009,6 +4009,23 @@ def test_scan_reads_statement_suites_an_annotation_runs(place, depth): ], describe(hits) +def test_scan_counts_levels_through_every_mix_of_code_calls(): + """Any ``exec`` or ``eval`` around a suite is one level, at any place.""" + for size, place in product(range(7), _FORCED_HINTS): + for names in product(("exec", "eval"), repeat=size): + code = _wrapped(_SUITE, 1) + for name in names: + code = f"{name}({code!r})" + text = f"{_IMPORT} as R\n{_FORCED_HINTS[place].format(code)}" + assert _answers(text) == [True, False] + line = 3 if place == "class" else 2 + kind = "isinstance" if size < 5 else "dynamic-code" + hits = scan_sources([_invented(text)]) + assert [(hit.line, hit.kind) for hit in hits] == [ + (line, kind) + ], describe(hits) + + _RUN_SUITE = _wrapped(_SUITE, 1) _RUN_DEEP_SUITE = _wrapped(_SUITE, 6) _ANNOTATED_SUITE = f"class C:\n v: {_RUN_SUITE}\nC.__annotations__\n" @@ -4030,6 +4047,10 @@ def test_scan_reads_statement_suites_an_annotation_runs(place, depth): f"class C:\n v: eval({_RUN_SUITE!r})\nC.__annotations__\n", [(3, "isinstance")], ), + "a padded expression literal around the suite": ( + f"class C:\n v: eval({' ' + _RUN_SUITE!r})\nC.__annotations__\n", + [(3, "isinstance")], + ), "a string hint": ( f"import typing\ndef g(v: {_RUN_SUITE!r}):\n pass\n" "typing.get_type_hints(g)\n", @@ -4173,51 +4194,51 @@ def _recursive_module_path(scope, node): return None -_PATH_BASES = st.sampled_from( - [ - "sys", - "os", - "engine", - "rng", - "unbound", - "__import__('sys')", - "__import__('os')", - "importlib.import_module('sys')", - "(lambda: sys)()", - "values[0]", - ] +_PATH_BASES = ( + "sys", + "os", + "engine", + "rng", + "exporter", + "unbound", + "__import__('sys')", + "__import__('')", + "importlib.import_module('sys')", + "(lambda: sys)()", + "values[0]", ) -_PATH_LINKS = st.lists( - st.sampled_from( - [ - ".modules['sys']", - ".modules['os']", - ".modules['']", - ".modules[key]", - ".modules['populace_dynamics.engine.rng']", - ".modules", - ".path", - ".rng", - ".ProjectionRNGRegistry", - "['sys']", - ".other['sys']", - "()", - ] - ), - max_size=8, +_PATH_LINKS = ( + ".modules['sys']", + ".modules['os']", + ".modules['']", + ".modules[key]", + ".modules['populace_dynamics.engine.rng']", + ".modules", + ".path", + ".rng", + ".alias", + ".ProjectionRNGRegistry", + "['sys']", + ".other['sys']", + "()", ) -@settings(max_examples=500, deadline=None) -@given(base=_PATH_BASES, links=_PATH_LINKS) -def test_module_path_agrees_with_a_recursive_walk(base, links): - """Unwinding a chain changes no path, and keeps its determinism.""" +def _path_scope(): + """Bind an invented module, with re-exports for ``_resolve`` to follow.""" source = _invented( "import importlib\nimport os\nimport sys\n" "import populace_dynamics.engine as engine\n" "from populace_dynamics.engine import rng\n" + "import invented_pkg.exporter as exporter\n" + ) + exports = { + ("populace_dynamics.engine", "rng"): "sys", + ("invented_pkg.exporter", "alias"): "sys", + } + scope = _ModuleBindings( + source, set(), set(), exports, _FoldMemo(source.path) ) - scope = _ModuleBindings(source, set(), set(), {}, _FoldMemo(source.path)) scope.bind( [ node @@ -4225,8 +4246,52 @@ def test_module_path_agrees_with_a_recursive_walk(base, links): if isinstance(node, _BINDING_NODES) ] ) + return scope + + +def _module_paths(scope, base, links): + """Give a chain's path by both walks, with a fresh memo for its nodes.""" chain = ast.parse(base + "".join(links), mode="eval").body - assert scope.module_path(chain) == _recursive_module_path(scope, chain) + scope.memo = _FoldMemo(scope.source.path) + return scope.module_path(chain), _recursive_module_path(scope, chain) + + +def test_module_path_agrees_with_a_recursive_walk_on_short_chains(): + """Unwinding changes no path on any chain of up to three links.""" + scope = _path_scope() + for size in range(4): + for links in product(_PATH_LINKS, repeat=size): + for base in _PATH_BASES: + unwound, recursive = _module_paths(scope, base, links) + assert unwound == recursive, (base, links) + + +@settings(max_examples=200, deadline=None) +@given( + base=st.sampled_from(_PATH_BASES), + links=st.lists(st.sampled_from(_PATH_LINKS), min_size=4, max_size=8), +) +def test_module_path_agrees_with_a_recursive_walk(base, links): + """Unwinding changes no path on a longer chain either.""" + unwound, recursive = _module_paths(_path_scope(), base, links) + assert unwound == recursive + + +@pytest.mark.parametrize( + "consumer", + ( + "import invented_pkg.exporter as e\nTARGET = e.alias", + "from invented_pkg.exporter import alias\nTARGET = alias", + ), +) +def test_scan_reads_sys_modules_through_a_re_exported_sys(consumer): + """A module's ``sys`` still reads ``sys.modules``, by name or attribute.""" + exporter = _invented("import sys as alias\n", "invented_pkg.exporter") + text = f"{consumer}.modules[{REGISTRY_MODULE!r}]\n" + hits = scan_sources([exporter, _invented(text)]) + assert [(hit.line, hit.kind) for hit in hits] == [ + (2, "module-reference") + ], describe(hits) _CHAIN = 3000 @@ -4287,12 +4352,12 @@ def test_module_path_agrees_with_a_recursive_walk(base, links): " return isinstance(x, ProjectionRNGRegistry) # check\n", "isinstance", ), - "a chain of sys.modules reads to the class": ( + "a chain of sys.modules reads to the module": ( "import sys\nimport populace_dynamics.engine.rng\n" - "def f(x):\n return isinstance(x, sys" + "def f(x):\n return isinstance(x, getattr(sys" + ".modules['sys']" * (_CHAIN // 2) - + ".modules['populace_dynamics.engine.rng'].ProjectionRNGRegistry)" - " # check\n", + + ".modules['populace_dynamics.engine.rng'], " + "'Projection' + 'RNGRegistry')) # check\n", "isinstance", ), } From 35ba6e5c4b2b95602b7b56858d062ff53326e712 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 14:02:24 -0500 Subject: [PATCH 4/9] Pin a code call with no argument in a hint Sweep two left one survivor: dropping the argument guard crashes the scan on exec() in a hint, and no test used that spelling. Co-Authored-By: Claude Opus 5.5 --- tests/open_population/test_stream_injection.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/open_population/test_stream_injection.py b/tests/open_population/test_stream_injection.py index 0c67a782..35019d2c 100644 --- a/tests/open_population/test_stream_injection.py +++ b/tests/open_population/test_stream_injection.py @@ -4127,6 +4127,8 @@ def test_pruning_keeps_each_hint_whole(place): "exec('pass')", "exec('x = 1')", "exec(code)", + "exec()", + "'exec()'", "eval('int')", "'int'", "eval('exec(\"pass\")')", From f8d64f8b2850791c46711b859212a396f129bf03 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 14:55:09 -0500 Subject: [PATCH 5/9] Record the follow-up's fixes and results in the report Co-Authored-By: Claude Opus 5.5 --- .../open_population_evidence/op1_report.md | 108 +++++++++++++++++- 1 file changed, 107 insertions(+), 1 deletion(-) diff --git a/docs/design/open_population_evidence/op1_report.md b/docs/design/open_population_evidence/op1_report.md index 30172c97..1f7cec20 100644 --- a/docs/design/open_population_evidence/op1_report.md +++ b/docs/design/open_population_evidence/op1_report.md @@ -1098,7 +1098,7 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at function's hint at the first and sixth levels. `test_pruning_keeps_each_hint_whole` puts a working check's literal under a lambda's default in each of the three annotation places. - - `test_scan_trusts_code_in_a_hint_that_reaches_no_class` gives six + - `test_scan_trusts_code_in_a_hint_that_reaches_no_class` gives eight hints in a module that binds nothing tracked, and `test_scan_leaves_expression_literals_in_hints_to_the_hint_checks` keeps `ForwardRef` and `eval` hints that name the registry trusted. @@ -1495,6 +1495,47 @@ section changes and the code does not: `tests/open_population` and mutant apart. **Fix:** the mutant paragraph now limits the claim to this repository's modules. +### Follow-up to review inject-r4 + +Review inject-r4's two minor findings first reached this report as wording, +and the section above records that wording as #591 landed it. This follow-up +branches from master `6413d842` and fixes both in the code, so the +`dynamic-code` bullet, limit class 4 and "Deep expressions" no longer state +either exception. The review's two nits stay stated limits. + +1. **Literal statement suites in annotations.** `_embedded` now passes each + hint to `_hint_code`. A code call whose literal parses as an expression + stays with the hint checks. Any other literal takes `_expanded`, the path + a code call outside an annotation takes, with the same memo, depth and + `unexpanded` set. `_hint_code` also follows an expression literal one + level deeper, to find a suite inside it. `hits()` already walked + annotation calls, so a trusted function's sixth-level call reports + `dynamic-code` with no change to `walked_hints`. The scan walks no + spelled copy, so the real hint stands for it in `unexpanded`, and `_kind` + now tests that set first, because a string hint is a constant. + + The fix needed two adjacent changes. `_prune` now keeps each hint whole: + with the old pruning, a pruned scan missed a suite that an unpruned scan + reported, in a literal under a call the folder does not fold through. A + code call on unreadable text spelled in a hint's literal, as in + `def g(v: 'exec(S)')`, now reports `dynamic-code` at the hint. Without + that, limit class 4 would need a new exception. +2. **`module_path` recursion.** One loop now unwinds attributes and + `sys.modules[...]` subscripts from the outermost link inward, and a + second replays the links from the base. A chain that raised + `RecursionError` at 1000 links at `6413d842` now passes at 3000. + +Six invariants hold for the fixes: +- A pruned scan and an unpruned scan give the same hits on every source. +- `module_path` agrees with a recursive reference on every chain the + recursion survives. +- A working check in a hint reports its own kind down to five levels and + `dynamic-code` past them, for every mix of `exec` and `eval`. +- A hint gives every check an `exec` statement gives, with the two + exceptions the parity test states. +- A literal in a hint that parses as an expression stays trusted. +- On the real tree the scan gives exactly the pinned hit. + ### Commands and results ```sh @@ -1723,6 +1764,63 @@ one-minute load under 100: `sys.modules` chain gives its check at 200 and 500 links and raises `RecursionError` at 1000 and 2000. +For the follow-up to review inject-r4, the main session ran the same +commands on the host at `35ba6e5c` from 19:11 to 19:53 UTC on 1 November, +against base `6413d842`. The next commit changes only this report. Each +process ran under the load-check audit hook and started only at a +one-minute load under 100: +- `tests/open_population`: **739 passed and 1 strict xfail** in 71.97 s. + The 45 tests this follow-up adds all sit in `test_stream_injection.py`. +- The other four modules: 104 passed in 4.22 s. +- The two Track B holdout-registry modules: 230 passed and 2 skipped. The + required command plus those two modules, in one process: **1,073 passed, + 2 skipped and 1 strict xfail**. +- Peak RSS was 1.35 GB. +- The slowest tests took 19.12 s (the real-tree scan), 4.58 s (the census) + and 4.31 s (a long chain). +- Nothing opened a file under a bar, the pattern line named only + `baselines/asfr.py` (read by the scan and the census, never displayed), + and a call-count hook on the one-process run counted no call to either + barred function. +- `--collect-only -m unit tests/open_population` collects all 740. Black, + Ruff and `git diff --check 6413d842` are clean. `git diff --stat + 6413d842` lists this report and `test_stream_injection.py`, and the + sealed-path `git diff --quiet` exits 0. +- On the real tree the scan gives exactly the pinned hit, the census pin + holds, and every module's folded values still weigh under 16,384; the + heaviest outside `baselines/asfr.py` weighs 10,435. Pruning still keeps + the 934,121 of 2,007,909 nodes the Cost bullet gives, so keeping hints + whole costs the real tree nothing. +- Through `repository_sources()`, the scans at `6413d842` and `35ba6e5c` + give the same single hit on all 302 modules. +- Pruned and unpruned, the two scans give the same hits on all 176 invented + sources at the base: the 125 spellings, the 20 negatives, the seven + accessor cases, nested depths one to seven, ten pruning places and seven + long chains. Of the 43 sources this follow-up adds, 16 give the same hits. + The 27 that differ are the intended ones: two `sys.modules` chains that + raised `RecursionError` and now give their checks, the twelve forced + annotations and the thirteen hint cases. The new scan's pruned and + unpruned hits agree on all 219. +- Review inject-r4's two minor findings no longer reproduce on invented + sources. The class and argument annotations that gave no hit at + `6413d842` report `isinstance` at `35ba6e5c`, and `sys.modules` chains + of 1000 and 2000 links that raised `RecursionError` report their check. + Its two nits reproduce unchanged, as stated limits. + +Thirty-nine mutants of the two fixes ran against the deterministic scan +tests, after an unmutated control. Twenty-eight touch the annotation fix: +a dropped hint pass, each annotation place and each pruning place dropped, +spelled parts unfollowed or uncounted, limits one higher and one lower, +wrong anchors, wrong line numbers, the unreadable call missed, and +expression literals expanded as code. Eleven touch `module_path`: no +`sys.modules` link, any attribute or module taken for it, a carried empty +path, links replayed in walk order, unresolved names and attributes, a lost +import base, the recursion restored, and a dropped link. The first sweep, of +36, left four survivors, and `9d81e218` adds the tests that kill them. The +second, of all 39 at `9d81e218`, left one: a code call with no argument in a +hint. `35ba6e5c` adds that spelling, and under that mutant it fails, so all +39 now fail a test. None of these ran against the real tree. + ### Restricted-files statement The build read `RESTRICTED-FILES.md` at its pinned hash and the @@ -1764,3 +1862,11 @@ changes used `--name-only`, `--stat` or `--quiet`, and displayed none of master's changed files beyond the two tier files. The real-tree scripts read the scanned modules only through the test module's own walk, under the audit hook, and printed paths, line numbers, node kinds and counts. + +The follow-up to review inject-r4 followed the same rules. The main session +read `RESTRICTED-FILES.md` at its pinned hash and the coordinator's five +pending entries before source work, and hashed them again before the review +brief. Its searches took their file lists from the helper, and its git +reads that spanned master's changes used `--name-only`, `--stat` or +`--quiet`. Every probe, differential source and mutant used invented +inputs. From 50e7e61ca4a1eed407dab511ba26c40b62de9538 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 2 Nov 2026 09:47:30 -0500 Subject: [PATCH 6/9] Bound the hint walk at five levels and read each text once Review scanfix-r1 found that the hint walk had no bound: a hint that spells itself ran the folder past its budget, and a 1002-level chain of forward references ended in a swallowed RecursionError and no hit. The walk now marks a sixth level of hint text as dynamic code, follows each text once per hint, as typing's recursive guard does, and skips the pieces a folded string is built from. Tests pin the quine, the countdown at 6, 50 and 1002 levels, the level boundary for strings and string hints, the data strings the scan reads on purpose, the pieces it skips, a name-bound hint it misses, and async return hints. The parity test now checks each lost hit's replacement on its line, and the mix test runs both scans. Co-Authored-By: Claude Opus 5.5 --- .../open_population/test_stream_injection.py | 273 ++++++++++++++++-- 1 file changed, 255 insertions(+), 18 deletions(-) diff --git a/tests/open_population/test_stream_injection.py b/tests/open_population/test_stream_injection.py index 35019d2c..b1b1c6d0 100644 --- a/tests/open_population/test_stream_injection.py +++ b/tests/open_population/test_stream_injection.py @@ -1240,22 +1240,42 @@ def _hint_code( memo: _FoldMemo, unexpanded: set[int], depth: int, + seen: set[str], ) -> list[ast.Module]: """Parse the statement suites a hint's code calls hold, nested too. - A literal that spells an expression is the hint checks' to read, so - the walk only follows it, one level deeper, for a suite inside. A code - call's other literal is code, as it is outside annotations. A spelled - part is no node the scan walks, so ``hint`` stands for it, and it goes - in ``unexpanded`` when such a part runs code the folder cannot read. + Python evaluates a string in a hint as the expression it spells, so + each string that spells one is a level of code, counted as a code + call's literal is. Its expression is the hint checks' to read, so the + walk only follows it, one level deeper, for a suite inside. A sixth + level is not followed: ``hint`` goes in ``unexpanded``. Nor is a text + the walk has followed before in this hint, in ``seen``, as ``typing`` + evaluates a forward reference once. The pieces the folder builds a + string from are not strings Python evaluates, so the walk skips them. + A code call's other literal is code, as it is outside annotations. A + spelled part is no node the scan walks, so ``hint`` stands for it, and + it goes in ``unexpanded`` when such a part runs code the folder cannot + read. """ found = [] - for node in ast.walk(part): - spelled = _spelled(node, memo) - if spelled is not None: + stack: list[ast.AST] = [part] + while stack: + node = stack.pop() + text = _static_text(node, memo) if isinstance(node, ast.expr) else None + if text is not None: + spelled = _spelled(node, memo) + if spelled is None or text.strip() in seen: + continue + seen.add(text.strip()) + if depth > _EMBED_DEPTH: + unexpanded.add(id(hint)) + continue _relocate(spelled, hint) - found += _hint_code(spelled, hint, memo, unexpanded, depth + 1) - elif ( + found += _hint_code( + spelled, hint, memo, unexpanded, depth + 1, seen + ) + continue + if ( isinstance(node, ast.Call) and node.args and _code_call(node.func) @@ -1264,6 +1284,7 @@ def _hint_code( if part is not hint and _static_text(node.args[0], memo) is None: unexpanded.add(id(hint)) found += _expanded(node, memo, unexpanded, depth, hint) + stack.extend(ast.iter_child_nodes(node)) return found @@ -1289,7 +1310,7 @@ def _embedded( ): found += _expanded(node, memo, unexpanded, depth, node) for hint in hints: - found += _hint_code(hint, hint, memo, unexpanded, depth) + found += _hint_code(hint, hint, memo, unexpanded, depth, set()) return found @@ -3978,6 +3999,7 @@ def test_scan_parses_five_levels_of_literal_code_and_flags_deeper(depth): "class": "class C:\n v: {}\nC.__annotations__\n", "argument": "def g(v: {}):\n pass\ng.__annotations__\n", "return": "def g() -> {}:\n pass\ng.__annotations__\n", + "async return": "async def g() -> {}:\n pass\ng.__annotations__\n", } @@ -4020,10 +4042,11 @@ def test_scan_counts_levels_through_every_mix_of_code_calls(): assert _answers(text) == [True, False] line = 3 if place == "class" else 2 kind = "isinstance" if size < 5 else "dynamic-code" - hits = scan_sources([_invented(text)]) - assert [(hit.line, hit.kind) for hit in hits] == [ - (line, kind) - ], describe(hits) + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == [ + (line, kind) + ], describe(hits) _RUN_SUITE = _wrapped(_SUITE, 1) @@ -4121,6 +4144,216 @@ def test_pruning_keeps_each_hint_whole(place): ], describe(hits) +# Line 9 of each module holds the hint, and f is no check until a hint's +# code redefines it. +_HINT_PREAMBLE = ( + f"import sys\nimport typing\n{_IMPORT} as R\nS = {_SUITE!r}\n" + "X = 'exec(S)'\nflag = 1\ndef f(x):\n return False\n" +) +_HINT_READ = "typing.get_type_hints(g)" +# Each level formats the next level's text, and the last spells exec(S). +_COUNTDOWN = ( + "('{{' + str({1})[:1] + '}}').format('', {0!r}, 'exec(S{{2}}')" + ".format({0!r}, {1} + 1, ')')" +) + + +def _hinted(hint, read=_HINT_READ): + """Put ``hint`` on line 9 of a module that reads it with ``read``.""" + return f"{_HINT_PREAMBLE}def g(v: {hint}):\n pass\n{read}\n" + + +def _strings(value, count): + """Nest ``value`` in ``count`` string literals.""" + for _ in range(count): + value = repr(value) + return value + + +def _countdown(levels): + """Return a string hint whose levels each spell the next, then run.""" + return repr(_COUNTDOWN.format(_COUNTDOWN, 2002 - levels)) + + +# Sources, what f answers once the hint is read, and the hits. +HINT_TEXT_CASES = { + "a string in a typing subscript": ( + _hinted(f"typing.List[{_RUN_SUITE!r}]"), + [True, False], + [(9, "isinstance")], + ), + "a string in a builtin generic": ( + _hinted(f"dict[{_RUN_SUITE!r}, 'int']"), + [True, False], + [(9, "isinstance")], + ), + "a string in a callable's arguments": ( + _hinted(f"typing.Callable[[{_RUN_SUITE!r}], int]"), + [True, False], + [(9, "isinstance")], + ), + "a string a conditional picks": ( + _hinted(f"{_RUN_SUITE!r} if flag else 'int'"), + [True, False], + [(9, "isinstance")], + ), + "a string an and picks": ( + _hinted(f"flag and {_RUN_SUITE!r}"), + [True, False], + [(9, "isinstance")], + ), + "a string a tuple index picks": ( + _hinted(f"({_RUN_SUITE!r}, 'int')[flag - 1]"), + [True, False], + [(9, "isinstance")], + ), + "four strings around a run": ( + _hinted(_strings(_RUN_SUITE, 4)), + [True, False], + [(9, "isinstance")], + ), + "five strings around a run": ( + _hinted(_strings(_RUN_SUITE, 5)), + [True, False], + [(9, "dynamic-code")], + ), + "a string hint around four execs": ( + _hinted(repr(_wrapped(_SUITE, 4))), + [True, False], + [(9, "isinstance")], + ), + "a string hint around five execs": ( + _hinted(repr(_wrapped(_SUITE, 5))), + [True, False], + [(9, "dynamic-code")], + ), + "five strings around a name": ( + _hinted(_strings("int", 5)), + [False, False], + [], + ), + "six strings around a name": ( + _hinted(_strings("int", 6)), + [False, False], + [(9, "dynamic-code")], + ), + "a countdown six levels deep": ( + _hinted(_countdown(6)), + [True, False], + [(9, "dynamic-code")], + ), + "a countdown 50 levels deep": ( + _hinted(_countdown(50)), + [True, False], + [(9, "dynamic-code")], + ), + "a countdown 1002 levels deep": ( + _hinted( + _countdown(1002), f"sys.setrecursionlimit(50000)\n{_HINT_READ}" + ), + [True, False], + [(9, "dynamic-code")], + ), + "a Literal value": ( + _hinted(f"typing.Literal[{_RUN_SUITE!r}]"), + [False, False], + [(9, "isinstance")], + ), + "Annotated metadata": ( + _hinted(f"typing.Annotated[int, {_RUN_SUITE!r}]"), + [False, False], + [(9, "isinstance")], + ), + "a list item": ( + _hinted(f"[{_RUN_SUITE!r}]", "g.__annotations__"), + [False, False], + [(9, "isinstance")], + ), + "a piece of a folded literal": ( + _hinted( + f"exec('x = ' + repr(repr({_RUN_SUITE!r})))", "g.__annotations__" + ), + [False, False], + [], + ), + "a piece of a spelled literal": ( + _hinted(f"{_RUN_SUITE!r}[:4]", "g.__annotations__"), + [False, False], + [], + ), + "a name bound to the text": (_hinted("X"), [True, False], []), +} + + +@pytest.mark.parametrize("case", tuple(HINT_TEXT_CASES)) +def test_scan_reads_each_string_in_a_hint_as_code(case): + """Each string in a hint that spells an expression is a level of code. + + Python evaluates a string in a typing subscript, and one a hint picks + at run time, so the scan reads every such string, to five levels, and + never the same text twice. So a string Python keeps as data gives a hit + too, and three cases pin that over-approximation. The scan reads no + piece of a folded or spelled literal, and no name bound to text: the + last case pins that miss, which limit class 1 states. + """ + text, answers, expected = HINT_TEXT_CASES[case] + limit = sys.getrecursionlimit() + try: + assert _answers(text) == answers + finally: + sys.setrecursionlimit(limit) + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == expected, describe( + hits + ) + + +@pytest.mark.parametrize("padding", (0, 600)) +def test_scan_follows_a_hint_text_once(padding): + """A hint that spells itself is read once, as ``typing`` reads it. + + The module's own check is the only hit. With 600 characters of padding, + following the text again at every level ran the folder past its budget. + """ + template = "{0!r}.format({0!r})" + " " * padding + text = ( + f"{_REGISTRY_IMPORT}" + f"def g(v: {template.format(template)}):\n pass\n" + "g.__annotations__\n" + "def f(x):\n return isinstance(x, ProjectionRNGRegistry)\n" + ) + assert _answers(text) == [True, False] + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == [ + (6, "isinstance") + ], describe(hits) + + +def test_scan_reads_a_text_again_in_another_hint(): + """Each hint has its own guard, so each line reports its own hit.""" + text = ( + f"import typing\n{_IMPORT} as R\ndef g(\n" + f" v: {_RUN_SUITE!r},\n w: {_RUN_SUITE!r},\n):\n pass\n" + "typing.get_type_hints(g)\n" + ) + assert _answers(text) == [True, False] + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == [ + (4, "isinstance"), + (5, "isinstance"), + ], describe(hits) + + +def test_scan_reports_a_sixth_level_of_hint_text_whatever_the_module_binds(): + """A sixth level is dynamic code, as a sixth code call's literal is.""" + text = f"def g(v: {_strings('int', 6)}):\n pass\n" + hits = scan_sources([_invented(text)]) + assert [(hit.line, hit.kind) for hit in hits] == [(1, "dynamic-code")] + + @pytest.mark.parametrize( "hint", ( @@ -4156,7 +4389,8 @@ def test_scan_reads_a_suite_in_a_hint_as_it_reads_one_in_a_statement(): So the hint gives every check the statement gives, with two exceptions. The catch-all reads the string in a statement, where it holds a tracked name, and skips it in a trusted annotation. And a hint that names a - tracked class makes a namespace read an annotation read. + tracked class makes a namespace read, or ``vars`` of a module, an + annotation read on the same line. """ def checks(text): @@ -4172,8 +4406,11 @@ def checks(text): for text in texts: plain = checks(f"exec({text!r})\n") hinted = checks(f"def g(v: exec({text!r})):\n pass\n") - added = {hit.kind for hit in hinted - plain} - assert plain <= hinted or added == {"annotation-read"} + reads = {hit.line for hit in hinted if hit.kind == "annotation-read"} + for hit in plain - hinted: + assert hit.kind in {"namespace-read", "module-reference"}, hit + assert hit.line in reads, hit + assert {hit.kind for hit in hinted - plain} <= {"annotation-read"} def _recursive_module_path(scope, node): From 7a830f39f352aecdb953202704fabc990276baa9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 2 Nov 2026 10:06:49 -0500 Subject: [PATCH 7/9] Report the bounded hint walk's results and review scanfix-r1's fixes Adds the host run, the differential against f8d64f8b and the 49-mutant sweep at 50e7e61c to the OP1 report, with review scanfix-r1's findings and the fix for each. Co-Authored-By: Claude Opus 5.5 --- .../open_population_evidence/op1_report.md | 196 ++++++++++++++++-- 1 file changed, 173 insertions(+), 23 deletions(-) diff --git a/docs/design/open_population_evidence/op1_report.md b/docs/design/open_population_evidence/op1_report.md index 1f7cec20..13beb9c4 100644 --- a/docs/design/open_population_evidence/op1_report.md +++ b/docs/design/open_population_evidence/op1_report.md @@ -921,15 +921,26 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at reported as `dynamic-code` whatever the module binds. A code call in an annotation follows the same rules when its literal does not parse as an expression, as in `v: exec("def f(x): ...")`, and the levels - count the same way. A literal that does parse as an expression stays - with the hint checks, so `ForwardRef("ProjectionRNGRegistry")` stays - trusted, and the scan looks inside it only for a suite. The scan - never asks whether Python evaluates the annotation, so a local - variable's hint, which Python never runs, gives the same hit. Where a - sixth level, or a code call on text the folder cannot read, sits in a - literal inside an annotation, the scan walks no node that holds it, so - the annotation itself reports `dynamic-code`, whatever the module - binds. + count the same way. In an annotation, a string that parses as an + expression is one more level of code, because `typing.get_type_hints` + evaluates a string hint, and a string in a subscript such as + `List["X"]`, as the expression it spells. The scan never parses such + a string as a suite: the hint checks read its expression, so + `ForwardRef("ProjectionRNGRegistry")` stays trusted. It does look + inside the expression for code calls and further strings. It reads + every such string in the hint, wherever it sits, because the folder + cannot always tell which one a hint picks at run time, as in + `"exec(S)" if flag else "int"`. So a string Python keeps as data there, + such as a `Literal` value, `Annotated` metadata or a list item, gives + the hit its code would give. The pieces the folder builds a string from + are not strings Python evaluates, and the scan skips them. It follows + each text once in a hint, as `typing` evaluates a forward reference + once, and it does not follow a string at a sixth level. The scan never + asks whether Python evaluates the annotation, so a local variable's + hint, which Python never runs, gives the same hit. Where a sixth level, + or a code call on text the folder cannot read, sits in a literal inside + an annotation, the scan walks no node that holds it, so the annotation + itself reports `dynamic-code`, whatever the module binds. - `member-introspection`: a dunder read on, or an introspection call such as `inspect.getmodule` or `getattr` on, a member of a module through which a class can be reached. @@ -1025,7 +1036,12 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at `test_module_path_agrees_with_a_recursive_walk` does the same on 200 Hypothesis chains of four to eight links, and `test_scan_reads_sys_modules_through_a_re_exported_sys` reaches a module - through a re-exported `sys`, by name and by attribute. + through a re-exported `sys`, by name and by attribute. The walk through + an annotation's strings recurses once per level, so it never goes past a + sixth level. Without that bound, a 1002-level chain of string hints, each + spelling the next, ended in a `RecursionError` that the folder absorbed, + and the scan gave no hit. A hint that spells itself, read once, can no + longer run the folder past its budget either. - **Cost.** Right after parsing, each literal container of constants is replaced by `None` unless a string in it holds either class's or module's name in any case, a probe name, or a name a read of a trusted @@ -1083,13 +1099,13 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at is reported as `isinstance` down to five levels and as `dynamic-code` at six and seven. - `test_scan_reads_statement_suites_an_annotation_runs` hides a working - check in class, argument and return annotations, in one, five, six and - seven literal `exec` calls, and reads the annotation to run it. Pruned - and unpruned, the scan reports `isinstance` down to five levels and - `dynamic-code` at six and seven. - `test_scan_counts_levels_through_every_mix_of_code_calls` does the same - for all 127 mixes of `exec` and `eval` in up to six outer calls, at each - of the three places. + check in class, argument, return and coroutine return annotations, in + one, five, six and seven literal `exec` calls, and reads the annotation + to run it. Pruned and unpruned, the scan reports `isinstance` down to + five levels and `dynamic-code` at six and seven. + `test_scan_counts_levels_through_every_mix_of_code_calls` does the same, + pruned and unpruned, for all 127 mixes of `exec` and `eval` in up to six + outer calls, at each of the four places. - `test_scan_reads_code_an_annotation_hides` gives thirteen more working checks: a folded literal, compiled code, an expression literal around the suite, a padded one, three string hints, a keyword-only and a @@ -1097,7 +1113,30 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at a coroutine, an annotation inside literal code, and a trusted function's hint at the first and sixth levels. `test_pruning_keeps_each_hint_whole` puts a working check's literal - under a lambda's default in each of the three annotation places. + under a lambda's default in each of the four annotation places. + - `test_scan_reads_each_string_in_a_hint_as_code` gives 21 cases, each + read the way it runs, pruned and unpruned: + - Working checks in a string in `List`, in `dict` and in a + `Callable`'s arguments, and in a string that a conditional, an `and` + or a tuple index picks at run time. + - The level boundary. Four strings around an `exec` give `isinstance` + and five give `dynamic-code`, as do a string hint around four and + five `exec` calls. Five strings around a name give nothing, and six + give `dynamic-code`. + - String-hint chains that each spell the next level, 6, 50 and 1002 + levels deep, all reported as `dynamic-code`. + - Three strings Python keeps as data, pinned as intended false hits: a + `Literal` value, `Annotated` metadata and a list item. + - Two pieces the scan skips, of a folded and of a spelled literal. + - One intended miss: a hint that names a variable bound to code + text (limit class 1). + + `test_scan_follows_a_hint_text_once` gives a hint that spells itself, + with and without 600 characters of padding, and expects only the + module's own check. `test_scan_reads_a_text_again_in_another_hint` + expects a hit on each of two hints that hold one text, and + `test_scan_reports_a_sixth_level_of_hint_text_whatever_the_module_binds` + reports six strings around a name in a module that binds nothing. - `test_scan_trusts_code_in_a_hint_that_reaches_no_class` gives eight hints in a module that binds nothing tracked, and `test_scan_leaves_expression_literals_in_hints_to_the_hint_checks` @@ -1106,8 +1145,9 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at wraps each spelling and negative in a hint and in an `exec` statement. The hint gives every check the statement gives, with two exceptions. The catch-all reads the statement's string, so its name hits do not - carry over. And a hint that names a tracked class makes a namespace - read an `annotation-read`. + carry over. And a hint that names a tracked class turns a namespace + read, or `vars` of a module, into an `annotation-read` on the same + line. The test checks each lost hit's replacement on its line. - **Its messages.** A hit carries only a module path, line number and node kind. `test_scan_messages_never_show_source_text` plants a distinctive literal in an invented snippet and checks that it is absent from the @@ -1127,7 +1167,10 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at attributes of objects built elsewhere, run-time containers, closures or frames: `type(a) is type(b)`, `isinstance(x, type(y))` with `y` passed in, and the dataclass-generated `==` between a registry and a - direct instance built elsewhere. + direct instance built elsewhere. So is a hint whose value is text the + folder cannot read, such as a name bound to a string, a call's result + or a literal expression past the folder's limits, which + `typing.get_type_hints` evaluates as a forward reference. 2. **Introspection of run-time values.** The annotations, docstring, source or members of an object that arrives at run time, such as a function passed in. Among the readers of run-time annotations in @@ -1533,9 +1576,64 @@ Six invariants hold for the fixes: `dynamic-code` past them, for every mix of `exec` and `eval`. - A hint gives every check an `exec` statement gives, with the two exceptions the parity test states. -- A literal in a hint that parses as an expression stays trusted. +- A literal in a hint that parses as an expression is never parsed as a + suite itself: the hint checks read it, and the scan looks inside it for + code calls and further strings. - On the real tree the scan gives exactly the pinned hit. +Review scanfix-r1's guard and bound add two more: +- The walk through a hint's strings follows at most five levels and each + text once, so it ends on every source. +- A string at a sixth level reports `dynamic-code`, whatever the module + binds. + +### Review scanfix-r1 + +Review scanfix-r1 read the follow-up at `f8d64f8b` and returned REVISE. A +Codex attempt stopped before its final checks, and a Claude Opus 5.5 attempt +finished the review. It found `module_path` correct: 374,348 exhaustive +chains of up to three links and 40,000 random longer ones matched the scan +at `6413d842`. It also found that the annotation fix reported every working +`exec` and `eval` check it built, 513 of them across 15 places. Its six +findings, and what changed: + +1. **Major: `_hint_code` followed spelled text with no bound.** A + self-spelling `str.format` hint of 2,011 characters, which the base read + correctly, ran the folder past its budget. A 1002-level chain of + string hints ended in a `RecursionError` that `_apply` absorbed, so the + scan gave no hit. **Fix:** the walk now puts a string at a sixth level + in `unexpanded` instead of following it, and each hint has a guard that + follows each text once, as `typing`'s recursive guard does. Both + sources are now tests, with the chain at 6, 50 and 1002 levels. +2. **Minor: data strings in a hint read as code.** A `Literal` value, + `Annotated` metadata, a list item, a `str` call's argument and a piece + of an `exec` literal each gave `isinstance`, though Python keeps them as + data. The review proposed following only strings in the places + `typing` evaluates. **Fix, in part:** the scan now skips the pieces of a + folded or spelled literal, which removes the hit from the piece of an + `exec` literal. The `str` call folds to a string hint, which + `get_type_hints` would run, so its hit stays. The scan keeps reading + every other string, because the folder cannot always tell which string + a hint picks at run time. A conditional, an `and` and a tuple + index with a run-time index each pick a string that `get_type_hints` + runs, and a walk over typing's places alone would miss all three + silently. The `dynamic-code` bullet now states the over-approximation, + invariant 5 says what the scan does with such a literal, and three + cases pin it as intended. +3. **Minor: no test had a coroutine's return hint**, and two mutants that + drop it survived. **Fix:** `_FORCED_HINTS` gains an async return place, + which the level, mix and pruning tests all use, and the sweep carries + both mutants. +4. **Minor: limit class 1 omitted hint values the folder cannot read.** + `typing.get_type_hints` runs `X = 'exec(S)'; def g(v: X)`, and neither + scan reports it. **Fix:** limit class 1 now states it, and a case pins + the miss. +5. **Nit: the parity test's `or` was looser than its claim.** **Fix:** it + now checks that each lost hit is a namespace read or `vars` of a module + with an `annotation-read` on its line, and that the hint adds only + `annotation-read`. §6 names both substitutions. +6. **Nit: the mix test ran only the pruned scan.** **Fix:** it runs both. + ### Commands and results ```sh @@ -1821,6 +1919,56 @@ second, of all 39 at `9d81e218`, left one: a code call with no argument in a hint. `35ba6e5c` adds that spelling, and under that mutant it fails, so all 39 now fail a test. None of these ran against the real tree. +After review scanfix-r1, the main session ran the same commands on the host +at `50e7e61c` from 14:50 to 15:02 UTC on 2 November, against base +`6413d842`. The next commit changes only this report. Each process ran +under the load-check audit hook and started only at a one-minute load under +100: +- `tests/open_population`: **769 passed and 1 strict xfail** in 60.37 s. + The 75 tests this follow-up adds all sit in `test_stream_injection.py`. +- The other four modules: 104 passed in 3.41 s. +- The two Track B holdout-registry modules: 230 passed and 2 skipped. The + required command plus those two modules, in one process: **1,103 passed, + 2 skipped and 1 strict xfail**. +- Peak RSS was 1.35 GB. +- The slowest tests took 16.03 s (the real-tree scan), 4.15 s (the census) + and 3.69 s (a long chain). +- Nothing opened a file under a bar, the pattern line named only + `baselines/asfr.py`, and a call-count hook on the one-process run counted + no call to either barred function. +- `--collect-only -m unit tests/open_population` collects all 770. Black, + Ruff and `git diff --check 6413d842` are clean, `git diff --stat + 6413d842` lists this report and `test_stream_injection.py`, and the + sealed-path `git diff --quiet` exits 0. +- On the real tree the scan gives exactly the pinned hit, the census pin + holds, the heaviest memo outside `baselines/asfr.py` weighs 10,435, and + pruning keeps 934,121 of 2,007,909 nodes. Through `repository_sources()` + the scans at `6413d842`, `f8d64f8b` and `50e7e61c` give the same single + hit on all 302 modules. +- Against `f8d64f8b`, pruned and unpruned, the scans give the same hits on + all 178 sources of the base corpus. Of the 579 sources this follow-up + adds, the 127 mixes at four places among them, six differ in both modes, + each as intended: + - Six strings around a name now give `dynamic-code`, as a sixth level. + - The 1002-level countdown now gives `dynamic-code` instead of nothing. + - The pieces of a folded and of a spelled literal now give nothing. + - Two quines padded to 600 and 1000 characters now give the module's own + hit instead of `ScanError`. +- The new scan's pruned and unpruned hits agree on all 757 sources. + +Forty-nine mutants ran against the deterministic scan tests at +`50e7e61c`, after an unmutated control. They repeat the sweep above, +whose `_hint_code` mutants now target the new walk, and add ten for review +scanfix-r1's fixes: +- no guard, a guard per level, and a guard shared across hints; +- no bound on hint text, and the bound one level early or late; +- folded pieces read, and spelled pieces read; +- r1's two async mutants, one dropping a coroutine's return hint and one + pruning it. + +All 49 fail a test, and the module came back byte for byte. None ran +against the real tree. + ### Restricted-files statement The build read `RESTRICTED-FILES.md` at its pinned hash and the @@ -1869,4 +2017,6 @@ pending entries before source work, and hashed them again before the review brief. Its searches took their file lists from the helper, and its git reads that spanned master's changes used `--name-only`, `--stat` or `--quiet`. Every probe, differential source and mutant used invented -inputs. +inputs. The fixes for review scanfix-r1 followed the same rules, after the +main session read the coordinator's sixth pending entry, which closes paths +that this branch's tree does not yet hold. From 181f40ca91aba390e12540ef6d45501181adec7a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 06:21:41 -0500 Subject: [PATCH 8/9] Read every string in a hint and give the hint walk its own budget Review scanfix-r2 found two major gaps. Skipping a folded subtree lost a working check under a rebound name, so the walk now reads every string and piece in a hint. A wide hint stopped the scan with the module's budget error, so a module's hints now share a second budget, and a hint the walk can't finish reports dynamic-code. A star text now reads as typing reads it. The report answers the minor finding and the nit, and states what stays. Co-Authored-By: Claude Opus 5.5 --- .../open_population_evidence/op1_report.md | 212 +++++++- .../open_population/test_stream_injection.py | 456 ++++++++++++++++-- 2 files changed, 613 insertions(+), 55 deletions(-) diff --git a/docs/design/open_population_evidence/op1_report.md b/docs/design/open_population_evidence/op1_report.md index 13beb9c4..479ffa60 100644 --- a/docs/design/open_population_evidence/op1_report.md +++ b/docs/design/open_population_evidence/op1_report.md @@ -932,15 +932,26 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at cannot always tell which one a hint picks at run time, as in `"exec(S)" if flag else "int"`. So a string Python keeps as data there, such as a `Literal` value, `Annotated` metadata or a list item, gives - the hit its code would give. The pieces the folder builds a string from - are not strings Python evaluates, and the scan skips them. It follows - each text once in a hint, as `typing` evaluates a forward reference - once, and it does not follow a string at a sixth level. The scan never - asks whether Python evaluates the annotation, so a local variable's - hint, which Python never runs, gives the same hit. Where a sixth level, - or a code call on text the folder cannot read, sits in a literal inside - an annotation, the scan walks no node that holds it, so the annotation - itself reports `dynamic-code`, whatever the module binds. + the hit its code would give. The scan also reads the pieces the folder + builds a string from. The folder takes `str` to mean the builtin, and a + module can bind that name to `eval`, so `(str("exec(S)"), "int")[1]` + can run a piece the folder's value leaves out. Reading every piece + gives a false hit where `str` keeps its meaning, as on a slice of a + literal, and the scan accepts that hit. A text that starts with `*` + counts as the `(*x,)[0]` that `typing` compiles from it. The walk + follows each distinct text once in a hint, which bounds its work. + `typing` guards a text only along one recursive path and evaluates it + again in a sibling position, so this guard rests on another argument: + the walk's first visit found what the text holds, or marked the hint. + The walk does not follow a string at a sixth level unless it has + followed that text before. A hit at a sixth level therefore depends on + the order: the walk reads a tuple's last item first, so a copy of the + text there hides the sixth level, having already read what it holds. + The scan never asks whether Python evaluates the annotation, so a local + variable's hint, which Python never runs, gives the same hit. Where a + sixth level, or a code call on text the folder cannot read, sits in a + literal inside an annotation, the scan walks no node that holds it, so + the annotation itself reports `dynamic-code`, whatever the module binds. - `member-introspection`: a dunder read on, or an introspection call such as `inspect.getmodule` or `getattr` on, a member of a module through which a class can be reached. @@ -989,7 +1000,19 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at text as long as that literal before the folder refuses the result. Across a module, the values the folder keeps weigh at most 1 MiB in characters and items, the weight of 16 texts at the limit; past that the - scan stops with an error naming the module, so it never skips one. + scan stops with an error naming the module, so it never skips one. The + walk through a module's hints spends from a second budget of the same + size, because most of what it reads Python never runs. All of a module's + hints share that budget, those inside what a hint runs included, since a + fresh budget for each nested hint would multiply the work at every level. + A hint the walk cannot finish reports `dynamic-code` in place of all the + walk found, and so does each later hint that holds an expression, because + the budget stays spent. The scan goes on and the module's other checks + still report. Those hits are an intended over-approximation: such a hint + may run no code at all. A suite a hint runs joins the module's code, so + the values the checks fold from it count against the module's budget. + Past that budget the scan stops with the error, as it does for the same + suite in a statement. - `test_folder_refuses_results_past_its_limits` checks 19 expressions past those limits: the folder gives no value, and holds under 1 MiB while refusing. @@ -1006,6 +1029,31 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at `test_scan_stops_at_the_folder_budget_naming_only_the_path`, a module of twelve built 64 KiB texts scans clean, one of twenty stops with the error, and one of twenty 64 KiB literals scans clean. + - `test_scan_reports_a_hint_past_the_walk_budget` puts a short hint, whose + 100 texts each spell a 4,000-character text, in all four annotation + places beside an ordinary check. Pruned and unpruned, the hint reports + `dynamic-code` and the check reports `isinstance`. + `test_scan_reports_a_hint_past_the_walk_budget_whatever_the_module_binds` + gives the same hint in a module that binds nothing tracked, and in a + statement's suite. + `test_scan_keeps_a_hint_within_the_walk_budget_off_the_module_budget` + gives a longer hint, of 70 texts, that fits the budget and scans clean. + `test_scan_shares_the_walk_budget_across_a_module` gives two hints that + each fit and together spend it: the second reports, and so does a + working check in a third. + `test_scan_bounds_hints_inside_what_a_hint_runs` gives hints that each + run a suite of 2, 4, 8 or 20 more such hints, and expects `dynamic-code` + in under 4,000 parses. + `test_a_hint_past_the_walk_budget_marks_only_itself` checks that the + failed walk leaves one mark, on the hint, and none on the tree it + dropped. + `test_a_failure_inside_what_a_hint_runs_goes_to_the_outermost_hint` gives + a hint that runs a suite whose own hint holds a text too deep for the + parser, before a working check: the outermost hint alone reports, and the + check reports through that hit. + `test_scan_stops_at_the_budget_on_a_suite_a_hint_runs` gives one + suite past the budget in a statement and in a hint, and expects the error + both ways. - `test_folder_runs_each_listed_method_and_builtin` calls each of the 26 methods and 10 builtins the folder runs (`format` both ways) and compares the result with Python's. @@ -1040,8 +1088,16 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at an annotation's strings recurses once per level, so it never goes past a sixth level. Without that bound, a 1002-level chain of string hints, each spelling the next, ended in a `RecursionError` that the folder absorbed, - and the scan gave no hit. A hint that spells itself, read once, can no - longer run the folder past its budget either. + and the scan gave no hit. A hint that spells itself takes one reading, so + it stays within the walk's budget. On this host Python's parser raises + `MemoryError` on a deep text, such as 10,000 unary operators, and + `RecursionError` on a very long chain, such as 100,000 additions. A text + that only the walk reads reports `dynamic-code`, as past the budget, in + `test_scan_reports_a_hint_text_too_deep_for_the_parser`. A code call and + the hint checks parse what they read, as they did at the base, so there + the error stops the scan, in + `test_scan_stops_at_a_text_too_deep_for_the_parser`. Python cannot compile + those texts either. - **Cost.** Right after parsing, each literal container of constants is replaced by `None` unless a string in it holds either class's or module's name in any case, a probe name, or a name a read of a trusted @@ -1114,23 +1170,35 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at function's hint at the first and sixth levels. `test_pruning_keeps_each_hint_whole` puts a working check's literal under a lambda's default in each of the four annotation places. - - `test_scan_reads_each_string_in_a_hint_as_code` gives 21 cases, each + - `test_scan_reads_each_string_in_a_hint_as_code` gives 34 cases, each read the way it runs, pruned and unpruned: - Working checks in a string in `List`, in `dict` and in a - `Callable`'s arguments, and in a string that a conditional, an `and` - or a tuple index picks at run time. + `Callable`'s arguments, in a string that a conditional, an `and` or a + tuple index picks at run time, in a string that starts with `*`, in + one with `*` only in a comment, and in a keyword argument. + - A working check whose text differs from a `Literal` value only in + case, so the guard must keep the two apart. + - Six working checks in a piece of a folded string: `str` or `repr` + bound to `eval`, `str` or another name bound to `exec`, and `str` + bound in literal code or through `globals()`. - The level boundary. Four strings around an `exec` give `isinstance` and five give `dynamic-code`, as do a string hint around four and five `exec` calls. Five strings around a name give nothing, and six give `dynamic-code`. + - The order at a sixth level. Six strings around a name give + `dynamic-code` before a copy of the name's text and nothing after + one, a copy with spaces around it included. - String-hint chains that each spell the next level, 6, 50 and 1002 levels deep, all reported as `dynamic-code`. - - Three strings Python keeps as data, pinned as intended false hits: a - `Literal` value, `Annotated` metadata and a list item. - - Two pieces the scan skips, of a folded and of a spelled literal. + - Five strings Python keeps as data, pinned as intended false hits: a + `Literal` value, `Annotated` metadata, a list item and the pieces of + a folded and of a spelled literal. - One intended miss: a hint that names a variable bound to code text (limit class 1). + `test_scan_reads_a_piece_when_builtins_itself_is_rebound` binds + `builtins.str` to `eval`, which no list of a module's own bindings would + see, and expects the piece's check. `test_scan_follows_a_hint_text_once` gives a hint that spells itself, with and without 600 characters of padding, and expects only the module's own check. `test_scan_reads_a_text_again_in_another_hint` @@ -1170,7 +1238,12 @@ of `src/populace_dynamics/` outside `open_population/`: 302 modules at direct instance built elsewhere. So is a hint whose value is text the folder cannot read, such as a name bound to a string, a call's result or a literal expression past the folder's limits, which - `typing.get_type_hints` evaluates as a forward reference. + `typing.get_type_hints` evaluates as a forward reference. So is a + callee the scan knows only by its name. After `run = exec`, `run(code)` + is no code call to the scan, and after `str = lambda _: code`, + `exec(str('pass'))` reads as `exec('pass')`. Each runs a working check + and gives no hit, in a statement or in a hint, and + `test_scan_reads_a_callee_by_its_name` pins all four misses. 2. **Introspection of run-time values.** The annotations, docstring, source or members of an object that arrives at run time, such as a function passed in. Among the readers of run-time annotations in @@ -1581,11 +1654,19 @@ Six invariants hold for the fixes: code calls and further strings. - On the real tree the scan gives exactly the pinned hit. -Review scanfix-r1's guard and bound add two more: +Reviews scanfix-r1 and scanfix-r2 add three more: - The walk through a hint's strings follows at most five levels and each - text once, so it ends on every source. + distinct text once, and all of a module's hints spend from one budget. Each + level parses no more text than the level above it holds, plus what that + budget lets the folder build, so the walk ends on every source within a + fixed multiple of the module's text and the budget. The walk itself never + stops the scan: past its budget, or on a text the parser refuses, the + outermost hint reports `dynamic-code`. - A string at a sixth level reports `dynamic-code`, whatever the module - binds. + binds, unless the walk has already followed that text in this hint. In + that case its first visit found what the text holds, or marked the hint. +- The walk reads every string in a hint, the pieces of a folded string + included, so no binding of a name the folder runs hides a literal from it. ### Review scanfix-r1 @@ -1603,8 +1684,8 @@ findings, and what changed: string hints ended in a `RecursionError` that `_apply` absorbed, so the scan gave no hit. **Fix:** the walk now puts a string at a sixth level in `unexpanded` instead of following it, and each hint has a guard that - follows each text once, as `typing`'s recursive guard does. Both - sources are now tests, with the chain at 6, 50 and 1002 levels. + follows each text once. Both sources are now tests, with the chain at 6, + 50 and 1002 levels. 2. **Minor: data strings in a hint read as code.** A `Literal` value, `Annotated` metadata, a list item, a `str` call's argument and a piece of an `exec` literal each gave `isinstance`, though Python keeps them as @@ -1619,7 +1700,8 @@ findings, and what changed: runs, and a walk over typing's places alone would miss all three silently. The `dynamic-code` bullet now states the over-approximation, invariant 5 says what the scan does with such a literal, and three - cases pin it as intended. + cases pin it as intended. Review scanfix-r2 found a working check that + the skip lost, so `7a830f39` is the last revision that skips pieces. 3. **Minor: no test had a coroutine's return hint**, and two mutants that drop it survived. **Fix:** `_FORCED_HINTS` gains an async return place, which the level, mix and pruning tests all use, and the sweep carries @@ -1634,6 +1716,75 @@ findings, and what changed: `annotation-read`. §6 names both substitutions. 6. **Nit: the mix test ran only the pruned scan.** **Fix:** it runs both. +### Review scanfix-r2 + +Review scanfix-r2, a delta review on a Codex lane, read the follow-up at +`7a830f39` and returned REVISE. It marked scanfix-r1's findings 3 to 6 +fixed, and it did not read `module_path` again. It ran 176 working mixes of +strings and `eval` at levels four to seven, and each kept the boundary at +five levels in both modes. Its four findings, the two mutants it added that +survived, and what changed: + +1. **Major: skipping a folded subtree lost a working check.** The folder + takes `str` to mean the builtin. After `str = eval`, the hint + `(str("exec(...)"), "int")[1]` folds to `"int"`, Python runs the piece, + and the skip left that piece unread. `f8d64f8b` reported the check, and + `7a830f39` and the base gave no hit. The review proposed that the folder + refuse any name the module rebinds. **Fix, by another route:** the walk + reads every string in a hint again, pieces included, and the folder stays + as it was. A list of bindings would miss `builtins.str = eval`, and it + would need its own rules for `globals()['str'] = eval`, + `exec('str = eval')` and a star import. It would also change what the + folder gives every other check in such a module. Reading every string catches + each of these with less code, and it costs a false hit on a piece, which + fails loudly. The two piece cases that `50e7e61c` moved to no hit go back + to `isinstance`, now pinned as false hits, and six working cases and + `test_scan_reads_a_piece_when_builtins_itself_is_rebound` pin the + checks. This reverses the part of scanfix-r1's finding 2 that skipped + pieces. The false hits that finding reported stand, as stated. +2. **Major: a wide hint stopped the scan.** A 6,429-character source of 100 + distinct texts, each spelling a 4,000-character text, ran the walk past + the module's budget. The base scanned it clean, and `f8d64f8b` and + `7a830f39` stopped with the error. A second source, of 70 texts under + three layers of `replace`, did the same. **Fix, in part as proposed:** the + walk now spends from a budget of its own, and a hint past it reports + `dynamic-code` while the module's checks go on. The folder bullet states + the over-approximation, and both sources are now tests, the first beside + an ordinary check and alone. The review proposed one budget per hint. A + first version of the fix did that, and a probe showed its cost: a hint + inside what a hint runs got a fresh budget, so a suite of six such hints, + 628 characters of source, took 46,659 parses, and the count grows with + the fifth power of the hints. So all of a module's hints share one + budget, a failure inside what a hint runs goes to the outermost hint, and + a test pins the parse count. Two neighbours of this fix needed a decision. + The `except` also takes the parser's `MemoryError` and `RecursionError`, + because at `7a830f39` the walk raised them on a second-level text past + the hint checks' 64 parts, where the base gave no hit. And a suite a hint + runs still counts against the module's budget, as the same suite in a + statement does, because the checks read that code and skipping it would + hide it. On that source this branch stops with the error where the base + gave no hit, and a test pins it. +3. **Minor: the sixth-level invariant omitted the guard.** `(six strings + around "int", "int")` gives no hit, because the walk reads the tuple's + last item first, and the reverse order gives `dynamic-code`. No working + check goes missing, since the first visit read the text. **Fix:** the + invariant and the bullet now state the exception, and three cases pin + the order. +4. **Nit: `typing`'s guard works along one recursive path.** `typing` ran a + sibling copy of one text twice, so the analogy claimed too much. **Fix:** + the bullet and the docstring now give the scan's own argument. +5. **Two mutants survived:** guard keys that fold case, and a walk that + skips keyword-argument children. **Fix:** a case for each, and both join + the sweep below. + +The main session found two more while answering these: +- A string that starts with `*` ran its check and gave no hit at the base + and at every revision since. `typing` compiles `*x` as `(*x,)[0]`, and the + parser refuses the bare form in expression mode. `_spelled` now reads it + the way `typing` does, so the hint checks read it too. +- A callee the scan knows only by name gave no hit at the base and at every + revision since. Limit class 1 now states it, with four pinned misses. + ### Commands and results ```sh @@ -2020,3 +2171,14 @@ reads that spanned master's changes used `--name-only`, `--stat` or inputs. The fixes for review scanfix-r1 followed the same rules, after the main session read the coordinator's sixth pending entry, which closes paths that this branch's tree does not yet hold. + +The fixes for review scanfix-r2 followed the same rules, after the main +session read the coordinator's seventh pending entry, which closes further +paths and opens none. They hold one departure, which review scanfix-r2's +report named. The real-tree figures helper printed one count about +`baselines/asfr.py` alone, its scan hits. The rule on that file bars any +figure about it alone. The line reached the main session's displays at +`35ba6e5c` and `50e7e61c`, and review scanfix-r2 read it in the helper's +output. It printed no source line, rate or table content, and this report +does not repeat it. The helper no longer prints that line, and the main +session removed it from its own saved outputs. diff --git a/tests/open_population/test_stream_injection.py b/tests/open_population/test_stream_injection.py index b1b1c6d0..cf59f4dd 100644 --- a/tests/open_population/test_stream_injection.py +++ b/tests/open_population/test_stream_injection.py @@ -946,6 +946,19 @@ def __init__(self, path: str = ""): super().__init__() self.path = path self.held = 0 + self._walk = None + + @property + def walk(self) -> _FoldMemo: + """Return the memo the walk through a module's hints spends from. + + The walk's own memo is its own walk, so a hint inside what a hint + runs spends from the same budget. + """ + if self._walk is None: + self._walk = _FoldMemo() + self._walk._walk = self._walk + return self._walk def _fold(node: ast.AST, memo: _FoldMemo): @@ -1197,12 +1210,19 @@ def _annotations(tree: ast.AST) -> list[ast.expr]: def _spelled(node: ast.AST, memo: _FoldMemo) -> ast.expr | None: - """Return the expression a node's text spells, if it spells one.""" + """Return the expression a node's text spells, if it spells one. + + ``typing`` compiles a text that starts with ``*`` as ``(*x,)[0]``, so + the parser reads it that way too. + """ text = _static_text(node, memo) if isinstance(node, ast.expr) else None if text is None: return None + text = text.strip() + if text.startswith("*"): + text = f"({text},)[0]" try: - return ast.parse(text.strip(), mode="eval").body + return ast.parse(text, mode="eval").body except (SyntaxError, ValueError): return None @@ -1249,32 +1269,30 @@ def _hint_code( call's literal is. Its expression is the hint checks' to read, so the walk only follows it, one level deeper, for a suite inside. A sixth level is not followed: ``hint`` goes in ``unexpanded``. Nor is a text - the walk has followed before in this hint, in ``seen``, as ``typing`` - evaluates a forward reference once. The pieces the folder builds a - string from are not strings Python evaluates, so the walk skips them. - A code call's other literal is code, as it is outside annotations. A - spelled part is no node the scan walks, so ``hint`` stands for it, and - it goes in ``unexpanded`` when such a part runs code the folder cannot - read. + the walk has followed before in this hint, in ``seen``, so the work + stays with the hint's distinct texts: the first visit found what the + text holds, or marked the hint. The walk also reads the pieces the + folder builds a text from, because the folder takes ``str`` to mean + the builtin, and a module may bind it to ``eval``. A code call's other + literal is code, as it is outside annotations. A spelled part is no + node the scan walks, so ``hint`` stands for it, and it goes in + ``unexpanded`` when such a part runs code the folder cannot read. """ found = [] stack: list[ast.AST] = [part] while stack: node = stack.pop() text = _static_text(node, memo) if isinstance(node, ast.expr) else None - if text is not None: - spelled = _spelled(node, memo) - if spelled is None or text.strip() in seen: - continue + spelled = None if text is None else _spelled(node, memo) + if spelled is not None and text.strip() not in seen: seen.add(text.strip()) if depth > _EMBED_DEPTH: unexpanded.add(id(hint)) - continue - _relocate(spelled, hint) - found += _hint_code( - spelled, hint, memo, unexpanded, depth + 1, seen - ) - continue + else: + _relocate(spelled, hint) + found += _hint_code( + spelled, hint, memo, unexpanded, depth + 1, seen + ) if ( isinstance(node, ast.Call) and node.args @@ -1297,6 +1315,15 @@ def _embedded( is code the module runs, so the scan checks it as part of the module. Inside an annotation the hint checks read literals that spell an expression, and ``_hint_code`` finds the rest. + + Most of what that walk reads Python never runs, so it spends from a + budget of its own, and the module's stays for what the module holds. All + of a module's hints share that budget, since a fresh one for each hint + inside what a hint runs would multiply the work at every level. A hint + the walk cannot finish, past the budget or on a text too deep for the + parser, goes in ``unexpanded`` in place of all the walk found, so no + mark outlives the tree it names. When the failure comes inside what a + hint runs, the outermost hint takes it. """ hints = _annotations(tree) hinted = {id(item) for hint in hints for item in ast.walk(hint)} @@ -1310,7 +1337,14 @@ def _embedded( ): found += _expanded(node, memo, unexpanded, depth, node) for hint in hints: - found += _hint_code(hint, hint, memo, unexpanded, depth, set()) + marks: set[int] = set() + try: + found += _hint_code(hint, hint, memo.walk, marks, depth, set()) + except (ScanError, MemoryError, RecursionError): + if memo.walk is memo: + raise + marks = {id(hint)} + unexpanded |= marks return found @@ -4158,9 +4192,12 @@ def test_pruning_keeps_each_hint_whole(place): ) -def _hinted(hint, read=_HINT_READ): - """Put ``hint`` on line 9 of a module that reads it with ``read``.""" - return f"{_HINT_PREAMBLE}def g(v: {hint}):\n pass\n{read}\n" +def _hinted(hint, read=_HINT_READ, bind=""): + """Put ``hint`` on line 9 of a module that reads it with ``read``. + + ``bind`` runs first, and moves the hint down by the lines it holds. + """ + return f"{_HINT_PREAMBLE}{bind}def g(v: {hint}):\n pass\n{read}\n" def _strings(value, count): @@ -4175,6 +4212,9 @@ def _countdown(levels): return repr(_COUNTDOWN.format(_COUNTDOWN, 2002 - levels)) +_PICKS_RUN = f"(str({_RUN_SUITE!r}), 'int')[1]" +_STATEMENT_PIECE = repr("pass\n") +_RUN_SUITE_LOWER = _RUN_SUITE.replace(", R)", ", r)") # Sources, what f answers once the hint is read, and the hits. HINT_TEXT_CASES = { "a string in a typing subscript": ( @@ -4207,6 +4247,74 @@ def _countdown(levels): [True, False], [(9, "isinstance")], ), + "a string that starts with a star": ( + _hinted(repr(f"*({_RUN_SUITE},)")), + [True, False], + [(9, "isinstance")], + ), + "a string with a star only in a comment": ( + _hinted(repr(f"{_RUN_SUITE} # *")), + [True, False], + [(9, "isinstance")], + ), + "a string in a keyword argument": ( + _hinted( + f"(lambda **kw: eval(kw['code']))(code={_RUN_SUITE!r})", + "g.__annotations__", + ), + [True, False], + [(9, "dynamic-code"), (9, "isinstance")], + ), + "a text that differs from another only in case": ( + _hinted( + f"typing.Tuple[{_RUN_SUITE!r}, " + f"typing.Literal[{_RUN_SUITE_LOWER!r}]]" + ), + [True, False], + [(9, "isinstance")], + ), + "a piece that str bound to eval runs": ( + _hinted(_PICKS_RUN, "g.__annotations__", "str = eval\n"), + [True, False], + [(10, "isinstance")], + ), + "a piece that repr bound to eval runs": ( + _hinted( + _PICKS_RUN.replace("str(", "repr("), + "g.__annotations__", + "repr = eval\n", + ), + [True, False], + [(10, "isinstance")], + ), + "a piece that str bound to exec runs": ( + _hinted( + f"str({_STATEMENT_PIECE} + {_RUN_SUITE!r})", + "g.__annotations__", + "str = exec\n", + ), + [True, False], + [(10, "isinstance")], + ), + "a piece that another name bound to exec runs": ( + _hinted( + f"run({_STATEMENT_PIECE} + {_RUN_SUITE!r})", + "g.__annotations__", + "run = exec\n", + ), + [True, False], + [(10, "isinstance")], + ), + "a piece that str bound in literal code runs": ( + _hinted(_PICKS_RUN, "g.__annotations__", "exec('str = eval')\n"), + [True, False], + [(10, "isinstance")], + ), + "a piece that str bound through globals runs": ( + _hinted(_PICKS_RUN, "g.__annotations__", "globals()['str'] = eval\n"), + [True, False], + [(9, "namespace-read"), (10, "isinstance")], + ), "four strings around a run": ( _hinted(_strings(_RUN_SUITE, 4)), [True, False], @@ -4237,6 +4345,21 @@ def _countdown(levels): [False, False], [(9, "dynamic-code")], ), + "a sixth level read before a copy of its text": ( + _hinted(f"('int', {_strings('int', 6)})", "g.__annotations__"), + [False, False], + [(9, "dynamic-code")], + ), + "a sixth level read after a copy of its text": ( + _hinted(f"({_strings('int', 6)}, 'int')", "g.__annotations__"), + [False, False], + [], + ), + "a sixth level read after a spaced copy of its text": ( + _hinted(f"({_strings('int', 6)}, ' int ')", "g.__annotations__"), + [False, False], + [], + ), "a countdown six levels deep": ( _hinted(_countdown(6)), [True, False], @@ -4274,12 +4397,12 @@ def _countdown(levels): f"exec('x = ' + repr(repr({_RUN_SUITE!r})))", "g.__annotations__" ), [False, False], - [], + [(9, "isinstance")], ), "a piece of a spelled literal": ( _hinted(f"{_RUN_SUITE!r}[:4]", "g.__annotations__"), [False, False], - [], + [(9, "isinstance")], ), "a name bound to the text": (_hinted("X"), [True, False], []), } @@ -4291,10 +4414,13 @@ def test_scan_reads_each_string_in_a_hint_as_code(case): Python evaluates a string in a typing subscript, and one a hint picks at run time, so the scan reads every such string, to five levels, and - never the same text twice. So a string Python keeps as data gives a hit - too, and three cases pin that over-approximation. The scan reads no - piece of a folded or spelled literal, and no name bound to text: the - last case pins that miss, which limit class 1 states. + never the same text twice. It reads the pieces of a folded literal too, + because a builtin the folder runs may be bound to ``eval``. So a string + Python keeps as data gives a hit, and five cases pin that + over-approximation. A text the walk has followed hides a sixth level of + that text, and three cases pin that the hit depends on the order. The + scan reads no name bound to text: the last case pins that miss, which + limit class 1 states. """ text, answers, expected = HINT_TEXT_CASES[case] limit = sys.getrecursionlimit() @@ -4309,9 +4435,263 @@ def test_scan_reads_each_string_in_a_hint_as_code(case): ) +def test_scan_reads_a_piece_when_builtins_itself_is_rebound(): + """No list of what a module binds would see ``builtins.str`` change.""" + text = _hinted( + _PICKS_RUN, + "g.__annotations__\nbuiltins.str = type('')", + "import builtins\nbuiltins.str = eval\n", + ) + original = builtins.str + try: + assert _answers(text) == [True, False] + finally: + builtins.str = original + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == [ + (11, "isinstance") + ], describe(hits) + + +# Working checks limit class 1 states: the scan reads a callee by its name. +CALLEES_BY_NAME = { + "a name bound to exec": f"run = exec\nrun({_SUITE!r})\n", + "a name bound to exec, in a hint": ( + f"run = exec\ndef g(v: run({_SUITE!r})):\n pass\n" + "g.__annotations__\n" + ), + "a builtin the folder runs, bound to return other text": ( + f"str = lambda _: {_SUITE!r}\nexec(str('pass'))\n" + ), + "the same builtin, in a hint": ( + f"str = lambda _: {_SUITE!r}\ndef g(v: exec(str('pass'))):\n pass\n" + "g.__annotations__\n" + ), +} + + +@pytest.mark.parametrize("case", tuple(CALLEES_BY_NAME)) +def test_scan_reads_a_callee_by_its_name(case): + """Each is an intended miss: the names are no code calls to the scan.""" + text = ( + f"{_IMPORT} as R\ndef f(x):\n return False\n{CALLEES_BY_NAME[case]}" + ) + assert _answers(text) == [True, False] + for prune in (True, False): + assert scan_sources([_invented(text)], prune=prune) == () + + +# Literals of texts the parser refuses, with MemoryError and RecursionError. +_TOO_DEEP = { + "a deep expression": repr("+" * 100_000 + "1"), + "a long chain": repr("1" + "+1" * 300_000), +} + + +def _wide_hint(count=100): + """Return a short hint whose texts each spell a 4,000-character one.""" + template = '{0!r}.replace("~", "~")' + texts = [ + repr(f"{template!r}.format({f'int # {index}'!r}.ljust(4000))") + for index in range(count) + ] + return f"({', '.join(texts)})" + + +@pytest.mark.parametrize("place", tuple(_FORCED_HINTS)) +def test_scan_reports_a_hint_past_the_walk_budget(place): + """The walk's values fill the walk's budget, and the module's stays. + + The module's own check still reports beside the hint. At ``7a830f39``, + whose walk spent the module's budget, this source stopped the scan, + though the hints run no code. + """ + hint = _FORCED_HINTS[place].format(_wide_hint()) + text = ( + f"{_REGISTRY_IMPORT}{hint}" + "def f(x):\n return isinstance(x, ProjectionRNGRegistry)\n" + ) + assert _answers(text) == [True, False] + line = 3 if place == "class" else 2 + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == [ + (line, "dynamic-code"), + (6, "isinstance"), + ], describe(hits) + + +@pytest.mark.parametrize("form", ("{0}", "exec({0!r})\n")) +def test_scan_reports_a_hint_past_the_walk_budget_whatever_the_module_binds( + form, +): + """A walk that stops is dynamic code, as a sixth level is. + + A hint in a statement's suite takes the same budget, and reports at the + statement. + """ + text = form.format(f"def g(v: {_wide_hint()}):\n pass\n") + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == [ + (1, "dynamic-code") + ], describe(hits) + + +def test_scan_shares_the_walk_budget_across_a_module(): + """Each of two hints fits the budget, and together they spend it. + + The second hint reports, and so does every hint after it, as the working + check in the last one shows. The first keeps its reading, and the checks' + lighter reading of the first two fits the module's budget. + """ + text = ( + f"{_IMPORT} as R\n" + f"def a(v: {_wide_hint(50)}):\n pass\n" + f"def b(v: {_wide_hint(50)}):\n pass\n" + f"def g(v: exec({_SUITE!r})):\n pass\n" + "g.__annotations__\n" + ) + assert _answers(text) == [True, False] + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == [ + (4, "dynamic-code"), + (6, "dynamic-code"), + ], describe(hits) + + +def _branching_hint(branches): + """Return a hint whose suite holds that many hints that do the same.""" + field = "{0!r}.format({0!r})" + hints = "".join( + f"def g{index}(v: exec({field})):\n pass\n" + for index in range(branches) + ) + return f"def g(v: exec({hints!r}.format({hints!r}))):\n pass\n" + + +@pytest.mark.parametrize("branches", (2, 4, 8, 20)) +def test_scan_bounds_hints_inside_what_a_hint_runs(branches, monkeypatch): + """Nested hints spend one budget, so the parses stay few at any width. + + A fresh budget for each nested hint would parse each level's suites + ``branches`` times over, which at 20 would take hours on a source of + under three kilobytes. Past the budget the outermost hint reports and + the scan drops what its walk found, so those trees never reach the + module's budget. + """ + parses = [] + real_parse = ast.parse + + def counted(*args, **kwargs): + parses.append(1) + return real_parse(*args, **kwargs) + + monkeypatch.setattr(ast, "parse", counted) + text = _branching_hint(branches) + for prune in (True, False): + parses.clear() + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == [ + (1, "dynamic-code") + ], describe(hits) + assert len(parses) < 4_000 + + +def test_scan_keeps_a_hint_within_the_walk_budget_off_the_module_budget(): + """The hint fits the walk's budget, and the checks fit the module's. + + At ``7a830f39`` the walk's values, added to what the hint checks read, + took the module past its budget. + """ + values = [] + for index in range(70): + expression = f"int # distinct {index} " + " " * 4000 + for _ in range(3): + expression = f'{expression!r}.replace("~", "~")' + values.append(repr(expression)) + text = f"def g(v: ({', '.join(values)})):\n pass\ng.__annotations__\n" + for prune in (True, False): + assert scan_sources([_invented(text)], prune=prune) == () + + +def test_a_hint_past_the_walk_budget_marks_only_itself(): + """A mark names a node the scan walks, never a tree the walk dropped. + + The walk reads the suite first and marks a code call past five levels, + and the budget then drops the tree that holds that call. + """ + suite = f"x = 1\n{_wrapped('pass', 6)}" + text = f"def g(v: ({_wide_hint()}, exec({suite!r}))):\n pass\n" + tree = ast.parse(text) + unexpanded = set() + assert _embedded(tree, _FoldMemo(), unexpanded) == [] + assert unexpanded == {id(_annotations(tree)[0])} + + +@pytest.mark.parametrize("deep", tuple(_TOO_DEEP)) +def test_a_failure_inside_what_a_hint_runs_goes_to_the_outermost_hint(deep): + """The outermost hint reports in place of all the walk found. + + A nested hint keeps no reading of its own, so the suite's check after it + reports through that one hit and never in its own name. The module is a + working check once its hint is read, and the hit covers it. + """ + suite = f"def h(w: {_TOO_DEEP[deep]}):\n pass\n{_SUITE}" + text = ( + f"{_IMPORT} as R\n" + f"def g(v: exec({suite!r})):\n pass\n" + "g.__annotations__\n" + ) + assert _answers(text) == [True, False] + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == [ + (2, "dynamic-code") + ], describe(hits) + + +@pytest.mark.parametrize("deep", tuple(_TOO_DEEP)) +def test_scan_reports_a_hint_text_too_deep_for_the_parser(deep): + """The walk reads a level the hint checks stop short of, past 64 parts. + + Python cannot compile these texts either, so the hit is a false one, but + it is a hit: the scan does not stop. + """ + padding = ", ".join(repr(f"int{index}") for index in range(70)) + text = f"def g(v: ({padding}, {_TOO_DEEP[deep]!r})):\n pass\n" + for prune in (True, False): + hits = scan_sources([_invented(text)], prune=prune) + assert [(hit.line, hit.kind) for hit in hits] == [ + (1, "dynamic-code") + ], describe(hits) + + +@pytest.mark.parametrize("deep", tuple(_TOO_DEEP)) +@pytest.mark.parametrize( + "form", + ("exec({})\n", "def g(v: {}):\n pass\n", "def g(v: {!r}):\n pass\n"), + ids=("a code call's literal", "a hint", "a hint's second level"), +) +def test_scan_stops_at_a_text_too_deep_for_the_parser(form, deep): + """Outside the walk's own reading, the parser's error stops the scan. + + A code call and the hint checks parse what they read, as they did at + the base. Python cannot compile these texts either, and the scan does + not pass the module. + """ + for prune in (True, False): + with pytest.raises((MemoryError, RecursionError)): + scan_sources( + [_invented(form.format(_TOO_DEEP[deep]))], prune=prune + ) + + @pytest.mark.parametrize("padding", (0, 600)) def test_scan_follows_a_hint_text_once(padding): - """A hint that spells itself is read once, as ``typing`` reads it. + """A hint that spells itself is read once, so the walk ends. The module's own check is the only hit. With 600 characters of padding, following the text again at every level ran the folder past its budget. @@ -5034,6 +5414,22 @@ def test_scan_stops_at_the_folder_budget_naming_only_the_path( ) +@pytest.mark.parametrize( + "form", ("exec({!r})\n", "def g(v: exec({!r})):\n pass\n") +) +def test_scan_stops_at_the_budget_on_a_suite_a_hint_runs(form): + """A suite a hint runs joins the module's code, as a statement's does.""" + suite = "".join( + f"TEXT_{index} = {_BUDGET_TEXTS['built']}\n" for index in range(20) + ) + with pytest.raises(ScanError) as raised: + scan_sources([_invented(form.format(suite))]) + assert str(raised.value) == ( + "invented/invented_case.py: literal expressions past the folder's " + "budget" + ) + + # One call to each method and builtin the folder runs; a name dropped from # its lists stops folding here. FOLD_CALLS = ( From 30b0df8133527af18e502d9871dc1a0282f22bed Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 6 Nov 2026 08:23:44 -0500 Subject: [PATCH 9/9] Record the host run at 181f40ca in the OP1 report The run from 12:36 to 13:19 UTC on 6 November, its 66 mutants, and the three newer pending entries. This commit changes only the report. Co-Authored-By: Claude Opus 5.5 --- .../open_population_evidence/op1_report.md | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/docs/design/open_population_evidence/op1_report.md b/docs/design/open_population_evidence/op1_report.md index 479ffa60..462d07d8 100644 --- a/docs/design/open_population_evidence/op1_report.md +++ b/docs/design/open_population_evidence/op1_report.md @@ -2120,6 +2120,71 @@ scanfix-r1's fixes: All 49 fail a test, and the module came back byte for byte. None ran against the real tree. +After review scanfix-r2, the main session ran the same commands on the host +at `181f40ca` from 12:36 to 13:19 UTC on 6 November, against base +`6413d842`. The next commit changes only this report. Each process ran +under the load-check audit hook. The first steps started only at a +one-minute load under 100, and the rest under 130, the mark the coordinator +set that day: +- `tests/open_population`: **812 passed and 1 strict xfail** in 169.14 s. + The 118 tests this follow-up adds all sit in `test_stream_injection.py`. +- The other four modules: 104 passed in 5.55 s. +- The two Track B holdout-registry modules: 230 passed and 2 skipped. The + required command plus those two modules, in one process: **1,146 passed, + 2 skipped and 1 strict xfail**. +- Peak RSS was 1.33 GB. +- The slowest tests took 46.67 s (the real-tree scan), 13.56 s (a long + chain) and 12.09 s (the census). +- Nothing opened a file under a bar, the pattern line named only + `baselines/asfr.py`, and a call-count hook on the one-process run counted + no call to either barred function. +- `--collect-only -m unit tests/open_population` collects all 813. Black, + Ruff and `git diff --check 6413d842` are clean, `git diff --stat + 6413d842` lists this report and `test_stream_injection.py`, and the + sealed-path `git diff --quiet` exits 0. +- On the real tree the scan gives exactly the pinned hit, the census pin + holds, every module's folded values weigh under 16,384, the heaviest memo + outside `baselines/asfr.py` weighs 10,433, the heaviest walk outside it + weighs 1,241, and pruning keeps 934,121 of 2,007,909 nodes. Through + `repository_sources()` the scans at `6413d842`, `f8d64f8b`, `7a830f39` + and `181f40ca` give the same single hit on all 302 modules. +- Against `7a830f39`, pruned and unpruned, the scans give the same hits on + all 178 sources of the base corpus. Of the 617 sources this follow-up + adds, 23 differ in both modes, each as intended: + - A string that starts with `*` now gives `isinstance`. + - Six pieces that a rebound name runs (`str` or `repr` bound to `eval`, + `str` or another name bound to `exec`, `str` bound in literal code and + through `globals`) now give `isinstance`. + - The pieces of a folded and of a spelled literal go back to `isinstance`. + - Six wide sources (four annotation places, nothing bound, in a + statement) and three branching ones now report `dynamic-code` instead + of `ScanError`, and the four annotation places also keep their + `isinstance`. + - `builtins` rebound now gives `isinstance`. + - Four sources too deep for the parser now give `dynamic-code` instead of + `MemoryError` or `RecursionError`. +- The new scan's pruned and unpruned hits agree on all 795 sources. + +Sixty-six mutants ran against the deterministic scan tests, after an +unmutated control (546 passed, 2 deselected), under a memory cap with +Hypothesis's explain phase off. They repeat the sweep above, with its two +piece mutants reversed, and add seventeen for review scanfix-r2's fixes: +- a walk that spends the module's budget, a fresh budget for each hint, a + nested hint keeping its own failure, and a walk's walk on another budget; +- a budget that stops the scan, and each parser error that stops it; +- a failure keeping the walk's marks, a failure marking nothing, and marks + dropped on success; +- guard keys that fold case or keep the raw text, and skipped keyword + children; +- no star form, and a star at any place; +- a parser error caught in the statement path or in `_spelled`. + +Sixty-five fail a test. The survivor, a nested hint keeping its own failure, +showed that no test pinned where a failure inside what a hint runs goes. +`test_a_failure_inside_what_a_hint_runs_goes_to_the_outermost_hint` now +kills it at the committed module (control 548 passed, 2 deselected). The +module came back byte for byte each time. None ran against the real tree. + ### Restricted-files statement The build read `RESTRICTED-FILES.md` at its pinned hash and the @@ -2182,3 +2247,7 @@ figure about it alone. The line reached the main session's displays at output. It printed no source line, rate or table content, and this report does not repeat it. The helper no longer prints that line, and the main session removed it from its own saved outputs. + +The main session read the coordinator's eighth, ninth and tenth pending +entries on 6 November. They close further paths, and none of those paths +sits in this branch's tree.