test: preserve optional-extra pytest collection

CHANGELOG.d/2.12.7-optional-extra-collection.md

@@ -0,0 +1,15 @@
+# Skip optional-extra suites when the sandbox did not install them
+
+- OpenCode coverage-evidence supplies pytest and coverage but not
+  LineageWeave's optional backend extras. Collection no longer fails
+  with ``ModuleNotFoundError`` for ``asyncpg``, ``psycopg2``, ``redis``,
+  ``fast_mlsirm``, or its ``numpy`` runtime dependency when those extras
+  are absent. The repository-root hook also covers sibling ``backend/tests``
+  and tests that import an optional dependency through a local module,
+  including ``period_report`` when only NumPy is unavailable.
+- Kept paths defer to pytest's remaining ignore hooks, preserving built-in
+  ``--ignore`` and project collection rules.
+- Hosted CI still installs ``dev`` and ``backend`` extras, so every
+  suite continues to collect and run there.
+- Login typecheck and the ADR 0001 gap baseline remain owned by
+  LineageWeave#426; this PR keeps only optional-extra collection work.

conftest.py

@@ -0,0 +1,21 @@
+"""Repository-wide pytest collection hooks for optional backend extras."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from lineageweave.optional_extra_collection import (
+    collection_path_requires_missing_extras,
+    missing_optional_extra_modules,
+)
+
+
+def pytest_ignore_collect(collection_path: Path, config: object) -> bool | None:
+    """Skip files that import optional extras the sandbox did not install."""
+    del config
+    if collection_path_requires_missing_extras(
+        collection_path,
+        missing_optional_extra_modules(),
+    ):
+        return True
+    return None

lineageweave/optional_extra_collection.py

@@ -0,0 +1,100 @@
+"""Decide which pytest files need optional extras that may be absent.
+
+OpenCode coverage-evidence runs in a networkless sandbox that supplies
+pytest and coverage but not LineageWeave's optional backend extras
+(``asyncpg``, ``psycopg2``, ``redis``, ``fast_mlsirm``, ``numpy``). Hosted CI
+installs those extras and collects every suite. This helper keeps
+collection from failing with ``ModuleNotFoundError`` when extras are
+absent, without skipping anything when they are present.
+"""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+from collections.abc import Iterable
+from pathlib import Path
+
+OPTIONAL_EXTRA_MODULES: tuple[str, ...] = (
+    "asyncpg",
+    "psycopg2",
+    "redis",
+    "fast_mlsirm",
+    "numpy",
+)
+
+_BACKEND_EXTRAS: frozenset[str] = frozenset(OPTIONAL_EXTRA_MODULES)
+_OPTIONAL_EXTRA_IMPORTERS: dict[str, tuple[str, ...]] = {
+    "asyncpg": (
+        "scripts.import_postgresql_posts",
+        "scripts.seed_demo_data",
+    ),
+    "fast_mlsirm": (
+        "lineageweave.period_report",
+        "lineageweave.post_evaluation",
+    ),
+    "numpy": ("lineageweave.period_report",),
+}
+_HELPER_TEST_NAME = "test_optional_extra_collection.py"
+
+
+def _imported_module_names(source: str) -> frozenset[str]:
+    """Return exact top-level module paths from syntactically valid imports."""
+    try:
+        tree = ast.parse(source)
+    except (SyntaxError, ValueError):
+        return frozenset()
+    imported: set[str] = set()
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            imported.update(alias.name for alias in node.names)
+        elif isinstance(node, ast.ImportFrom) and node.module:
+            imported.add(node.module)
+    return frozenset(imported)
+
+
+def missing_optional_extra_modules(
+    module_names: Iterable[str] = OPTIONAL_EXTRA_MODULES,
+) -> tuple[str, ...]:
+    """Return optional extra names the current interpreter cannot import."""
+    return tuple(
+        name for name in module_names if importlib.util.find_spec(name) is None
+    )
+
+
+def collection_path_requires_missing_extras(
+    collection_path: Path,
+    missing_modules: Iterable[str],
+) -> bool:
+    """Return whether collecting this path would import a missing extra."""
+    missing = tuple(missing_modules)
+    if not missing:
+        return False
+    if collection_path.suffix != ".py":
+        return False
+    if collection_path.name == _HELPER_TEST_NAME:
+        return False
+    posix = collection_path.as_posix()
+    if any(name in _BACKEND_EXTRAS for name in missing) and (
+        posix == "backend" or posix.startswith("backend/") or "/backend/" in posix
+    ):
+        return True
+    try:
+        text = collection_path.read_text(encoding="utf-8")
+    except (OSError, UnicodeError):
+        return False
+    imported_modules = _imported_module_names(text)
+    for name in missing:
+        for imported_name in (name, *_OPTIONAL_EXTRA_IMPORTERS.get(name, ())):
+            if any(
+                module_name == imported_name
+                or module_name.startswith(f"{imported_name}.")
+                for module_name in imported_modules
+            ):
+                return True
+        if name in _BACKEND_EXTRAS and any(
+            module_name == "backend" or module_name.startswith("backend.")
+            for module_name in imported_modules
+        ):
+            return True
+    return False

tests/test_optional_extra_collection.py

@@ -0,0 +1,170 @@
+"""Unit tests for optional-extra pytest collection skipping."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from unittest.mock import patch
+
+import conftest as root_conftest
+from lineageweave.optional_extra_collection import (
+    OPTIONAL_EXTRA_MODULES,
+    collection_path_requires_missing_extras,
+    missing_optional_extra_modules,
+)
+
+
+def test_missing_optional_extra_modules_returns_absent_names() -> None:
+    """Names whose find_spec is None are reported; present names are not."""
+
+    def fake_find_spec(name: str) -> object | None:
+        """Model one unavailable optional module and one installed module."""
+        if name == "asyncpg":
+            return None
+        return object()
+
+    with patch(
+        "lineageweave.optional_extra_collection.importlib.util.find_spec",
+        side_effect=fake_find_spec,
+    ):
+        assert missing_optional_extra_modules(("asyncpg", "redis")) == ("asyncpg",)
+
+
+def test_collection_path_does_not_skip_when_no_extras_are_missing(
+    tmp_path: Path,
+) -> None:
+    """Hosted CI with extras installed still collects every suite."""
+    path = tmp_path / "test_api.py"
+    path.write_text("import asyncpg\n", encoding="utf-8")
+    assert collection_path_requires_missing_extras(path, ()) is False
+
+
+def test_collection_path_skips_backend_tree_when_asyncpg_is_missing(
+    tmp_path: Path,
+) -> None:
+    """Backend tests import the FastAPI app, which imports asyncpg at module level."""
+    backend_test = tmp_path / "backend" / "tests" / "test_api.py"
+    backend_test.parent.mkdir(parents=True)
+    backend_test.write_text("import pytest\n", encoding="utf-8")
+    assert collection_path_requires_missing_extras(backend_test, ("asyncpg",)) is True
+    assert collection_path_requires_missing_extras(backend_test, ("fast_mlsirm",)) is True
+    assert collection_path_requires_missing_extras(backend_test, ("numpy",)) is True
+
+
+def test_collection_path_skips_files_that_import_a_missing_extra(
+    tmp_path: Path,
+) -> None:
+    """A test that imports fast_mlsirm is ignored only when that extra is absent."""
+    path = tmp_path / "test_post_evaluation.py"
+    path.write_text("from fast_mlsirm import LLMJudgeResult\n", encoding="utf-8")
+    assert collection_path_requires_missing_extras(path, ("fast_mlsirm",)) is True
+    assert collection_path_requires_missing_extras(path, ("asyncpg",)) is False
+
+
+def test_collection_path_does_not_match_comments_or_import_prefixes(
+    tmp_path: Path,
+) -> None:
+    """Only parsed imports may suppress collection in the reduced sandbox."""
+    path = tmp_path / "test_unrelated.py"
+    path.write_text(
+        '"""Example text: import asyncpg."""\n'
+        "import rediscache\n"
+        "from backends import client\n",
+        encoding="utf-8",
+    )
+    assert collection_path_requires_missing_extras(
+        path, ("asyncpg", "redis")
+    ) is False
+
+
+def test_collection_path_skips_known_transitive_optional_importers(
+    tmp_path: Path,
+) -> None:
+    """Known local modules propagate their hard optional import."""
+    period_report = tmp_path / "test_period_report.py"
+    period_report.write_text(
+        "from lineageweave.period_report import build_period_report\n",
+        encoding="utf-8",
+    )
+    post_import = tmp_path / "test_import_postgresql_posts.py"
+    post_import.write_text(
+        "from scripts.import_postgresql_posts import parse_args\n",
+        encoding="utf-8",
+    )
+    assert (
+        collection_path_requires_missing_extras(period_report, ("fast_mlsirm",))
+        is True
+    )
+    assert collection_path_requires_missing_extras(period_report, ("numpy",)) is True
+    assert (
+        collection_path_requires_missing_extras(period_report, ("asyncpg",)) is False
+    )
+    post_evaluation = tmp_path / "test_post_evaluation.py"
+    post_evaluation.write_text(
+        "from lineageweave.post_evaluation import LLMJudgeResult\n",
+        encoding="utf-8",
+    )
+    assert (
+        collection_path_requires_missing_extras(post_evaluation, ("numpy",)) is False
+    )
+    backend_import = tmp_path / "test_report_ingestion.py"
+    backend_import.write_text(
+        "from backend.app.report_ingestion import ingest_report\n",
+        encoding="utf-8",
+    )
+    assert collection_path_requires_missing_extras(backend_import, ("asyncpg",)) is True
+    assert collection_path_requires_missing_extras(post_import, ("asyncpg",)) is True
+    seed = tmp_path / "test_seed.py"
+    seed.write_text("from scripts.seed_demo_data import seed\n", encoding="utf-8")
+    assert collection_path_requires_missing_extras(seed, ("redis",)) is False
+
+
+def test_helper_test_module_is_never_ignored(tmp_path: Path) -> None:
+    """The collection-helper tests must run in the sandbox that lacks extras."""
+    path = tmp_path / "test_optional_extra_collection.py"
+    path.write_text("import asyncpg\n", encoding="utf-8")
+    assert collection_path_requires_missing_extras(path, OPTIONAL_EXTRA_MODULES) is False
+
+
+def test_non_python_paths_are_not_ignored(tmp_path: Path) -> None:
+    """Collection directories and non-Python files stay visible to pytest."""
+    directory = tmp_path / "tests"
+    directory.mkdir()
+    assert collection_path_requires_missing_extras(directory, ("asyncpg",)) is False
+
+
+def test_unreadable_or_invalid_python_does_not_suppress_collection(
+    tmp_path: Path,
+) -> None:
+    """Collection errors stay visible instead of being hidden as missing extras."""
+    invalid = tmp_path / "test_invalid.py"
+    invalid.write_text("from asyncpg import\n", encoding="utf-8")
+    assert collection_path_requires_missing_extras(invalid, ("asyncpg",)) is False
+
+    unreadable = tmp_path / "test_unreadable.py"
+    with patch.object(Path, "read_text", side_effect=OSError("unreadable")):
+        assert (
+            collection_path_requires_missing_extras(unreadable, ("asyncpg",))
+            is False
+        )
+
+    non_utf8 = tmp_path / "test_non_utf8.py"
+    non_utf8.write_bytes(b"\xff")
+    assert collection_path_requires_missing_extras(non_utf8, ("asyncpg",)) is False
+
+
+def test_root_hook_defers_kept_paths_to_other_pytest_ignore_rules(
+    tmp_path: Path,
+) -> None:
+    """The first-result hook ignores optional paths and defers kept paths."""
+    optional_test = tmp_path / "test_optional.py"
+    optional_test.write_text("import asyncpg\n", encoding="utf-8")
+    ordinary_test = tmp_path / "test_ordinary.py"
+    ordinary_test.write_text("import pytest\n", encoding="utf-8")
+
+    with patch.object(
+        root_conftest,
+        "missing_optional_extra_modules",
+        return_value=("asyncpg",),
+    ):
+        assert root_conftest.pytest_ignore_collect(optional_test, object()) is True
+        assert root_conftest.pytest_ignore_collect(ordinary_test, object()) is None