fix(deps): sync requirements files with pyproject and guard the drift

Dockerfile.base

@@ -24,7 +24,11 @@ RUN set -eux && \
     /var/lang/bin/python3.12 -m venv /opt/venv && \
     /opt/venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel && \
     /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'numpy>=1.26.0,<2.0' && \
-    /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'tiktoken<0.8.0' && \
+    # litellm (pulled in by jvagent/pageindex) requires tiktoken>=0.8.0,<1.0, and
+    # pageindex itself declares >=0.11.0 — the old 'tiktoken<0.8.0' cap could not
+    # satisfy either. cp312 manylinux wheels exist across this range, so
+    # --only-binary still resolves.
+    /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'tiktoken>=0.11.0,<1.0' && \
     /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'pydantic[email]' && \
     # /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'docling>=2.0.0' && \
     # /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'tabulate>=0.9.0' && \

requirements-all.txt

@@ -3,13 +3,23 @@
 # Install with: pip install -r requirements-all.txt
 
 # Core jvagent dependencies
-jvspatial==0.0.15
+# Must stay in sync with [project] dependencies in pyproject.toml —
+# enforced by tests/test_requirements_sync.py.
+aiohttp>=3.9.0
+jvspatial==0.0.16
 python-dotenv>=1.0.0
 pyyaml>=6.0.0
 httpx>=0.27.0
 jinja2>=3.1.0
+pymupdf>=1.24.0
+packaging>=21.0
+mcp>=1.0.0
+
+# Dependencies from action info.yaml files.
+# Every pip dep an action declares must appear here — enforced by
+# tests/test_requirements_sync.py. Where actions declare different floors for
+# the same package, the highest wins (that is what pip resolves to anyway).
 
-# Dependencies from action info.yaml files
 # From jvagent/typesense_vectorstore
 typesense>=2.0.0
 
@@ -19,3 +29,35 @@ openai>=1.0.0
 # Additional dependencies for email validation support
 # pydantic[email] provides email-validator for EmailStr type support
 pydantic[email]>=2.13.4
+
+# From jvagent/web_fetch
+beautifulsoup4
+markdownify
+
+# From jvagent/stt_action/deepgram and jvagent/tts_action/elevenlabs
+deepgram-sdk>=6.0.0
+elevenlabs>=1.13.0
+
+# From jvagent/whatsapp
+filetype>=1.2.0
+
+# From the jvagent/google/* actions and jvagent/pageindex_google_drive_sync_action
+google-api-python-client>=2.192.0
+google-auth-httplib2>=0.3.0
+google-auth-oauthlib>=1.3.0
+
+# From jvagent/web_search/serpapi
+google-search-results>=2.4.2
+
+# From jvagent/pageindex/pageindex_action
+litellm>=1.82.0
+pypdf>=4.0.0
+
+# From jvagent/microsoft/microsoft_excel_action
+openpyxl>=3.1.0
+
+# From jvagent/facebook_action
+requests>=2.28.0
+
+# From jvagent/pageindex (also required by litellm: >=0.8.0,<1.0)
+tiktoken>=0.11.0

requirements.txt

@@ -1,7 +1,13 @@
-# Core runtime. Test-only deps (incl. Docling for PageIndex): pyproject.toml
+# Core runtime. Must stay in sync with [project] dependencies in pyproject.toml
+# — enforced by tests/test_requirements_sync.py.
+# Test-only deps (incl. Docling for PageIndex): pyproject.toml
 # [project.optional-dependencies] test — install with: pip install -e ".[test]"
-jvspatial==0.0.15
+aiohttp>=3.9.0
+jvspatial==0.0.16
 python-dotenv>=1.0.0
 pyyaml>=6.0.0
 httpx>=0.27.0
 jinja2>=3.1.0
+pymupdf>=1.24.0
+packaging>=21.0
+mcp>=1.0.0

tests/test_requirements_sync.py

@@ -0,0 +1,151 @@
+"""The requirements files must match ``[project] dependencies`` in pyproject.
+
+``pyproject.toml`` is the source of truth for what jvagent needs at runtime, but
+two requirements files are what actually get installed in places pip's metadata
+never reaches:
+
+- ``Dockerfile.base`` builds the runtime image from ``requirements-all.txt``.
+- ``requirements.txt`` is what the README and runbooks tell people to install.
+
+Nothing kept them in step, and they drifted: both sat two jvspatial releases
+behind, and both were missing four core dependencies outright (aiohttp, pymupdf,
+packaging, mcp), so ``pip install -r requirements.txt`` produced an install that
+could not load several actions. The failure is silent at install time and only
+shows up as an ImportError deep in a run.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Dict, List
+
+import pytest
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+PYPROJECT = REPO_ROOT / "pyproject.toml"
+REQUIREMENTS = ("requirements.txt", "requirements-all.txt")
+
+# Packages an action declares that requirements-all.txt deliberately omits.
+# Each needs a reason; empty is the goal, and it is currently empty.
+ACTION_DEP_EXCEPTIONS: set = set()
+
+_REQUIREMENT_RE = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*(?:\[[^\]]+\])?)\s*(.*)$")
+
+
+def _canonical(name: str) -> str:
+    """PEP 503 normalization, minus any extras marker."""
+    return re.sub(r"[-_.]+", "-", name.split("[")[0]).lower()
+
+
+def _core_dependencies() -> Dict[str, str]:
+    """``{canonical_name: full_spec}`` from ``[project] dependencies``."""
+    text = PYPROJECT.read_text(encoding="utf-8")
+    block = re.search(r"^dependencies = \[(.*?)^\]", text, re.S | re.M)
+    assert block, "could not locate [project] dependencies in pyproject.toml"
+    deps: Dict[str, str] = {}
+    for raw in re.findall(r'"([^"]+)"', block.group(1)):
+        match = _REQUIREMENT_RE.match(raw)
+        assert match, f"unparsable dependency spec: {raw!r}"
+        deps[_canonical(match.group(1))] = raw.strip()
+    return deps
+
+
+def _listed(filename: str) -> Dict[str, str]:
+    """``{canonical_name: full_spec}`` for one requirements file."""
+    listed: Dict[str, str] = {}
+    for line in (REPO_ROOT / filename).read_text(encoding="utf-8").splitlines():
+        line = line.split("#")[0].strip()
+        if not line or line.startswith("-"):
+            continue
+        match = _REQUIREMENT_RE.match(line)
+        if match:
+            listed[_canonical(match.group(1))] = line
+    return listed
+
+
+def test_core_dependencies_are_listed() -> None:
+    """Every runtime dependency must appear in both requirements files."""
+    core = _core_dependencies()
+    missing: List[str] = []
+    for filename in REQUIREMENTS:
+        listed = _listed(filename)
+        for name in core:
+            if name not in listed:
+                missing.append(f"{filename}: {core[name]}")
+    assert not missing, "missing from requirements files:\n  " + "\n  ".join(missing)
+
+
+def _action_pip_dependencies() -> Dict[str, List[str]]:
+    """``{canonical_name: [declaring info.yaml, ...]}`` across every action."""
+    declared: Dict[str, List[str]] = {}
+    for info in sorted(REPO_ROOT.glob("jvagent/action/**/info.yaml")):
+        try:
+            data = yaml.safe_load(info.read_text(encoding="utf-8")) or {}
+        except yaml.YAMLError:  # pragma: no cover - malformed yaml is its own bug
+            continue
+        package = data.get("package")
+        if not isinstance(package, dict):
+            continue
+        deps = package.get("dependencies")
+        pips = deps.get("pip") if isinstance(deps, dict) else None
+        if isinstance(pips, dict):
+            pips = [f"{k}{v}" for k, v in pips.items()]
+        for spec in pips or []:
+            if not isinstance(spec, str) or not spec.strip():
+                continue
+            match = _REQUIREMENT_RE.match(spec.strip())
+            if match:
+                name = _canonical(match.group(1))
+                declared.setdefault(name, []).append(str(info.relative_to(REPO_ROOT)))
+    return declared
+
+
+def test_action_dependencies_are_in_requirements_all() -> None:
+    """requirements-all.txt must carry every pip dep an action declares.
+
+    Its own header promises "all core dependencies plus all optional
+    dependencies from action info.yaml files", and Dockerfile.base builds the
+    runtime image from it — so anything missing is an action that cannot run in
+    the shipped image.
+    """
+    declared = _action_pip_dependencies()
+    assert declared, "found no action pip dependencies — parser is broken"
+    listed = _listed("requirements-all.txt")
+    missing = {
+        name: sources
+        for name, sources in declared.items()
+        if name not in listed and name not in ACTION_DEP_EXCEPTIONS
+    }
+    detail = "\n  ".join(
+        f"{name} (declared by {sources[0]}"
+        + (f" +{len(sources) - 1} more)" if len(sources) > 1 else ")")
+        for name, sources in sorted(missing.items())
+    )
+    assert not missing, "action deps missing from requirements-all.txt:\n  " + detail
+
+
+@pytest.mark.parametrize("name", sorted(ACTION_DEP_EXCEPTIONS))
+def test_exceptions_are_still_declared_somewhere(name: str) -> None:
+    """Keep the allowlist honest — drop entries once the action stops needing them."""
+    assert name in _action_pip_dependencies(), (
+        f"{name} is allowlisted in ACTION_DEP_EXCEPTIONS but no action declares "
+        "it any more; remove the exception."
+    )
+
+
+def test_core_dependency_specs_match() -> None:
+    """A listed dependency must carry the same version spec as pyproject.
+
+    A requirements file pinning an older jvspatial than pyproject declares is
+    how the Docker image and the tested tree end up on different versions.
+    """
+    core = _core_dependencies()
+    mismatched: List[str] = []
+    for filename in REQUIREMENTS:
+        for name, spec in _listed(filename).items():
+            expected = core.get(name)
+            if expected is not None and spec != expected:
+                mismatched.append(f"{filename}: {spec!r} != pyproject {expected!r}")
+    assert not mismatched, "version specs out of sync:\n  " + "\n  ".join(mismatched)