diff --git a/Dockerfile.base b/Dockerfile.base index e1e1228b..b827bf35 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -24,7 +24,11 @@ RUN set -eux && \ /var/lang/bin/python3.12 -m venv /opt/venv && \ /opt/venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel && \ /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'numpy>=1.26.0,<2.0' && \ - /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'tiktoken<0.8.0' && \ + # litellm (pulled in by jvagent/pageindex) requires tiktoken>=0.8.0,<1.0, and + # pageindex itself declares >=0.11.0 — the old 'tiktoken<0.8.0' cap could not + # satisfy either. cp312 manylinux wheels exist across this range, so + # --only-binary still resolves. + /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'tiktoken>=0.11.0,<1.0' && \ /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'pydantic[email]' && \ # /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'docling>=2.0.0' && \ # /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'tabulate>=0.9.0' && \ diff --git a/requirements-all.txt b/requirements-all.txt index 83c7a972..e6402c4e 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -3,13 +3,23 @@ # Install with: pip install -r requirements-all.txt # Core jvagent dependencies -jvspatial==0.0.15 +# Must stay in sync with [project] dependencies in pyproject.toml — +# enforced by tests/test_requirements_sync.py. +aiohttp>=3.9.0 +jvspatial==0.0.16 python-dotenv>=1.0.0 pyyaml>=6.0.0 httpx>=0.27.0 jinja2>=3.1.0 +pymupdf>=1.24.0 +packaging>=21.0 +mcp>=1.0.0 + +# Dependencies from action info.yaml files. +# Every pip dep an action declares must appear here — enforced by +# tests/test_requirements_sync.py. Where actions declare different floors for +# the same package, the highest wins (that is what pip resolves to anyway). -# Dependencies from action info.yaml files # From jvagent/typesense_vectorstore typesense>=2.0.0 @@ -19,3 +29,35 @@ openai>=1.0.0 # Additional dependencies for email validation support # pydantic[email] provides email-validator for EmailStr type support pydantic[email]>=2.13.4 + +# From jvagent/web_fetch +beautifulsoup4 +markdownify + +# From jvagent/stt_action/deepgram and jvagent/tts_action/elevenlabs +deepgram-sdk>=6.0.0 +elevenlabs>=1.13.0 + +# From jvagent/whatsapp +filetype>=1.2.0 + +# From the jvagent/google/* actions and jvagent/pageindex_google_drive_sync_action +google-api-python-client>=2.192.0 +google-auth-httplib2>=0.3.0 +google-auth-oauthlib>=1.3.0 + +# From jvagent/web_search/serpapi +google-search-results>=2.4.2 + +# From jvagent/pageindex/pageindex_action +litellm>=1.82.0 +pypdf>=4.0.0 + +# From jvagent/microsoft/microsoft_excel_action +openpyxl>=3.1.0 + +# From jvagent/facebook_action +requests>=2.28.0 + +# From jvagent/pageindex (also required by litellm: >=0.8.0,<1.0) +tiktoken>=0.11.0 diff --git a/requirements.txt b/requirements.txt index 53550d56..670de4d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,13 @@ -# Core runtime. Test-only deps (incl. Docling for PageIndex): pyproject.toml +# Core runtime. Must stay in sync with [project] dependencies in pyproject.toml +# — enforced by tests/test_requirements_sync.py. +# Test-only deps (incl. Docling for PageIndex): pyproject.toml # [project.optional-dependencies] test — install with: pip install -e ".[test]" -jvspatial==0.0.15 +aiohttp>=3.9.0 +jvspatial==0.0.16 python-dotenv>=1.0.0 pyyaml>=6.0.0 httpx>=0.27.0 jinja2>=3.1.0 +pymupdf>=1.24.0 +packaging>=21.0 +mcp>=1.0.0 diff --git a/tests/test_requirements_sync.py b/tests/test_requirements_sync.py new file mode 100644 index 00000000..87e99b73 --- /dev/null +++ b/tests/test_requirements_sync.py @@ -0,0 +1,151 @@ +"""The requirements files must match ``[project] dependencies`` in pyproject. + +``pyproject.toml`` is the source of truth for what jvagent needs at runtime, but +two requirements files are what actually get installed in places pip's metadata +never reaches: + +- ``Dockerfile.base`` builds the runtime image from ``requirements-all.txt``. +- ``requirements.txt`` is what the README and runbooks tell people to install. + +Nothing kept them in step, and they drifted: both sat two jvspatial releases +behind, and both were missing four core dependencies outright (aiohttp, pymupdf, +packaging, mcp), so ``pip install -r requirements.txt`` produced an install that +could not load several actions. The failure is silent at install time and only +shows up as an ImportError deep in a run. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Dict, List + +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +PYPROJECT = REPO_ROOT / "pyproject.toml" +REQUIREMENTS = ("requirements.txt", "requirements-all.txt") + +# Packages an action declares that requirements-all.txt deliberately omits. +# Each needs a reason; empty is the goal, and it is currently empty. +ACTION_DEP_EXCEPTIONS: set = set() + +_REQUIREMENT_RE = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*(?:\[[^\]]+\])?)\s*(.*)$") + + +def _canonical(name: str) -> str: + """PEP 503 normalization, minus any extras marker.""" + return re.sub(r"[-_.]+", "-", name.split("[")[0]).lower() + + +def _core_dependencies() -> Dict[str, str]: + """``{canonical_name: full_spec}`` from ``[project] dependencies``.""" + text = PYPROJECT.read_text(encoding="utf-8") + block = re.search(r"^dependencies = \[(.*?)^\]", text, re.S | re.M) + assert block, "could not locate [project] dependencies in pyproject.toml" + deps: Dict[str, str] = {} + for raw in re.findall(r'"([^"]+)"', block.group(1)): + match = _REQUIREMENT_RE.match(raw) + assert match, f"unparsable dependency spec: {raw!r}" + deps[_canonical(match.group(1))] = raw.strip() + return deps + + +def _listed(filename: str) -> Dict[str, str]: + """``{canonical_name: full_spec}`` for one requirements file.""" + listed: Dict[str, str] = {} + for line in (REPO_ROOT / filename).read_text(encoding="utf-8").splitlines(): + line = line.split("#")[0].strip() + if not line or line.startswith("-"): + continue + match = _REQUIREMENT_RE.match(line) + if match: + listed[_canonical(match.group(1))] = line + return listed + + +def test_core_dependencies_are_listed() -> None: + """Every runtime dependency must appear in both requirements files.""" + core = _core_dependencies() + missing: List[str] = [] + for filename in REQUIREMENTS: + listed = _listed(filename) + for name in core: + if name not in listed: + missing.append(f"{filename}: {core[name]}") + assert not missing, "missing from requirements files:\n " + "\n ".join(missing) + + +def _action_pip_dependencies() -> Dict[str, List[str]]: + """``{canonical_name: [declaring info.yaml, ...]}`` across every action.""" + declared: Dict[str, List[str]] = {} + for info in sorted(REPO_ROOT.glob("jvagent/action/**/info.yaml")): + try: + data = yaml.safe_load(info.read_text(encoding="utf-8")) or {} + except yaml.YAMLError: # pragma: no cover - malformed yaml is its own bug + continue + package = data.get("package") + if not isinstance(package, dict): + continue + deps = package.get("dependencies") + pips = deps.get("pip") if isinstance(deps, dict) else None + if isinstance(pips, dict): + pips = [f"{k}{v}" for k, v in pips.items()] + for spec in pips or []: + if not isinstance(spec, str) or not spec.strip(): + continue + match = _REQUIREMENT_RE.match(spec.strip()) + if match: + name = _canonical(match.group(1)) + declared.setdefault(name, []).append(str(info.relative_to(REPO_ROOT))) + return declared + + +def test_action_dependencies_are_in_requirements_all() -> None: + """requirements-all.txt must carry every pip dep an action declares. + + Its own header promises "all core dependencies plus all optional + dependencies from action info.yaml files", and Dockerfile.base builds the + runtime image from it — so anything missing is an action that cannot run in + the shipped image. + """ + declared = _action_pip_dependencies() + assert declared, "found no action pip dependencies — parser is broken" + listed = _listed("requirements-all.txt") + missing = { + name: sources + for name, sources in declared.items() + if name not in listed and name not in ACTION_DEP_EXCEPTIONS + } + detail = "\n ".join( + f"{name} (declared by {sources[0]}" + + (f" +{len(sources) - 1} more)" if len(sources) > 1 else ")") + for name, sources in sorted(missing.items()) + ) + assert not missing, "action deps missing from requirements-all.txt:\n " + detail + + +@pytest.mark.parametrize("name", sorted(ACTION_DEP_EXCEPTIONS)) +def test_exceptions_are_still_declared_somewhere(name: str) -> None: + """Keep the allowlist honest — drop entries once the action stops needing them.""" + assert name in _action_pip_dependencies(), ( + f"{name} is allowlisted in ACTION_DEP_EXCEPTIONS but no action declares " + "it any more; remove the exception." + ) + + +def test_core_dependency_specs_match() -> None: + """A listed dependency must carry the same version spec as pyproject. + + A requirements file pinning an older jvspatial than pyproject declares is + how the Docker image and the tested tree end up on different versions. + """ + core = _core_dependencies() + mismatched: List[str] = [] + for filename in REQUIREMENTS: + for name, spec in _listed(filename).items(): + expected = core.get(name) + if expected is not None and spec != expected: + mismatched.append(f"{filename}: {spec!r} != pyproject {expected!r}") + assert not mismatched, "version specs out of sync:\n " + "\n ".join(mismatched)