From f1cec83177a46d39ac4f95bfbd2677e8877407c9 Mon Sep 17 00:00:00 2001 From: Liam Huber Date: Sun, 6 Sep 2026 21:56:59 -0700 Subject: [PATCH] Allow compiling inside modules that are still importing By modifying how `object_scope` resolves attributes to object. Co-authored-by: Claude Signed-off-by: Liam Huber --- src/flowrep/parsers/object_scope.py | 20 +++-- .../test_building_while_importing.py | 74 +++++++++++++++++++ tests/unit/parsers/test_object_scope.py | 23 ++++++ 3 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 tests/integration/test_building_while_importing.py diff --git a/src/flowrep/parsers/object_scope.py b/src/flowrep/parsers/object_scope.py index d3704316..f6445266 100644 --- a/src/flowrep/parsers/object_scope.py +++ b/src/flowrep/parsers/object_scope.py @@ -112,13 +112,21 @@ def resolve_attribute_to_object(attribute: str, scope: ScopeProxy | object) -> o Returns: The object that the attribute resolves to in the given scope. """ - obj = None - try: - for attr in attribute.split("."): + obj, prefix = None, [] + for attr in attribute.split("."): + prefix.append(attr) + try: obj = getattr(obj or scope, attr) - return obj - except AttributeError as e: - raise ValueError(f"Could not find attribute '{attr}' of {attribute}") from e + except AttributeError as e: + # A package's attribute for a still-executing submodule isn't set until + # that import finishes; sys.modules already has it. + try: + obj = sys.modules[".".join(prefix)] + except KeyError: + raise ValueError( + f"Could not find attribute '{attr}' of {attribute}" + ) from e + return obj def resolve_symbol_to_object( diff --git a/tests/integration/test_building_while_importing.py b/tests/integration/test_building_while_importing.py new file mode 100644 index 00000000..77a117a4 --- /dev/null +++ b/tests/integration/test_building_while_importing.py @@ -0,0 +1,74 @@ +"""Compiling and building a workflow from inside a module that is *still importing*. + +``flowrep2python`` emits fully-qualified child calls (``pkg.mod.add(...)``), and +``RenderedSource.build`` re-parses that emitted source, so the parser has to resolve +``pkg.mod.add`` by walking attributes down from ``pkg``. CPython only sets the +submodule attribute on a parent package once the submodule's body has finished +executing, so a module that compiles one of its own workflows at import time hands the +parser a package that does not yet know about it -- ``getattr(pkg, "mod")`` raises +``AttributeError: cannot access submodule 'mod' of module 'pkg'``. + +``sys.modules`` holds the entry from the moment the import begins, which is what +``object_scope.resolve_attribute_to_object`` falls back on. +""" + +import importlib +import pathlib +import sys +import tempfile +import textwrap +import unittest + +_PACKAGE = "_flowrep_still_importing_pkg" + +_MODULE_SOURCE = """ +import flowrep + + +@flowrep.atomic("z") +def add(x, y): + z = x + y + return z + + +@flowrep.workflow("out") +def wf(x, y): + s = add(x=x, y=y) + return s + + +# Compiling here, at module scope, is the whole point of the fixture: the parent +# package has no attribute for this module yet. +recipe = wf.flowrep_recipe.model_copy(update={"reference": None}) +built = flowrep.tools.flowrep2python(recipe, function_name="wf").build() +""" + + +class TestBuildingWhileImporting(unittest.TestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + package = pathlib.Path(self._tmp.name) / _PACKAGE + package.mkdir() + (package / "__init__.py").touch() + (package / "mod.py").write_text(textwrap.dedent(_MODULE_SOURCE)) + sys.path.insert(0, self._tmp.name) + importlib.invalidate_caches() + + def tearDown(self): + sys.path.remove(self._tmp.name) + for name in [n for n in sys.modules if n.split(".")[0] == _PACKAGE]: + del sys.modules[name] + self._tmp.cleanup() + + def test_child_reference_resolves_before_the_import_finishes(self): + module = importlib.import_module(f"{_PACKAGE}.mod") + self.assertEqual( + 3, + module.built(x=1, y=2), + msg="The re-parsed source must reach `add` through the half-imported " + "package and still describe the same computation.", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/parsers/test_object_scope.py b/tests/unit/parsers/test_object_scope.py index ff0f8d94..d0e2f49e 100644 --- a/tests/unit/parsers/test_object_scope.py +++ b/tests/unit/parsers/test_object_scope.py @@ -281,6 +281,29 @@ def test_resolve_attribute_to_object(self): f = object_scope.resolve_attribute_to_object("ast.literal_eval", scope) self.assertIs(f, ast.literal_eval) + def test_submodule_still_importing_resolves_via_sys_modules(self): + """A parent package gains its submodule attribute only once that submodule + has finished importing, but ``sys.modules`` carries the entry from the start. + Anything that resolves a dotted name while the submodule is mid-import -- such + as re-parsing compiled source at module scope -- sees the gap.""" + parent = types.ModuleType("_test_half_imported_pkg") + child = types.ModuleType("_test_half_imported_pkg.child") + marker = object() + child.marker = marker # type: ignore[attr-defined] + sys.modules[parent.__name__] = parent + sys.modules[child.__name__] = child + # Deliberately *not* `parent.child = child`: that is what the import machinery + # does last, after the child's body has run to completion. + try: + scope = object_scope.ScopeProxy({parent.__name__: parent}) + resolved = object_scope.resolve_attribute_to_object( + f"{child.__name__}.marker", scope + ) + self.assertIs(marker, resolved) + finally: + del sys.modules[child.__name__] + del sys.modules[parent.__name__] + if __name__ == "__main__": unittest.main()