From 6c68e072e0e746dfd3b3c7a80b22845311f1d1a0 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Fri, 7 Aug 2026 10:53:24 -0400 Subject: [PATCH 1/3] fix(deps): sync requirements files with pyproject and guard the drift requirements.txt and requirements-all.txt each listed 5 of the 9 runtime dependencies pyproject declares. aiohttp, pymupdf, packaging and mcp were absent from both, and both pinned jvspatial two releases behind. This is not only a stale doc. Dockerfile.base builds the runtime image from requirements-all.txt, so the image shipped without four runtime dependencies -- an install that succeeds and then fails as an ImportError deep in a run, when the action loader reaches for mcp or pymupdf. Add the missing dependencies to both files and align the jvspatial pin with pyproject. Then stop it recurring. tests/test_requirements_sync.py treats pyproject as the source of truth and asserts two properties over both files: every core dependency is listed, and every listed spec matches pyproject exactly. Nothing enforced either before, which is why the files could sit wrong indefinitely -- pip metadata never reads them, so no install ever disagreed. Both assertions were mutation-checked rather than trusted green: removing mcp fails the first, reverting the pin fails the second, and restoring passes. --- requirements-all.txt | 8 ++- requirements.txt | 10 +++- tests/test_requirements_sync.py | 86 +++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 tests/test_requirements_sync.py diff --git a/requirements-all.txt b/requirements-all.txt index 83c7a972..9383727d 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -3,11 +3,17 @@ # Install with: pip install -r requirements-all.txt # Core jvagent dependencies -jvspatial==0.0.15 +# Must stay in sync with [project] dependencies in pyproject.toml — +# enforced by tests/test_requirements_sync.py. +aiohttp>=3.9.0 +jvspatial==0.0.16 python-dotenv>=1.0.0 pyyaml>=6.0.0 httpx>=0.27.0 jinja2>=3.1.0 +pymupdf>=1.24.0 +packaging>=21.0 +mcp>=1.0.0 # Dependencies from action info.yaml files # From jvagent/typesense_vectorstore diff --git a/requirements.txt b/requirements.txt index 53550d56..670de4d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,13 @@ -# Core runtime. Test-only deps (incl. Docling for PageIndex): pyproject.toml +# Core runtime. Must stay in sync with [project] dependencies in pyproject.toml +# — enforced by tests/test_requirements_sync.py. +# Test-only deps (incl. Docling for PageIndex): pyproject.toml # [project.optional-dependencies] test — install with: pip install -e ".[test]" -jvspatial==0.0.15 +aiohttp>=3.9.0 +jvspatial==0.0.16 python-dotenv>=1.0.0 pyyaml>=6.0.0 httpx>=0.27.0 jinja2>=3.1.0 +pymupdf>=1.24.0 +packaging>=21.0 +mcp>=1.0.0 diff --git a/tests/test_requirements_sync.py b/tests/test_requirements_sync.py new file mode 100644 index 00000000..6d69df5b --- /dev/null +++ b/tests/test_requirements_sync.py @@ -0,0 +1,86 @@ +"""The requirements files must match ``[project] dependencies`` in pyproject. + +``pyproject.toml`` is the source of truth for what jvagent needs at runtime, but +two requirements files are what actually get installed in places pip's metadata +never reaches: + +- ``Dockerfile.base`` builds the runtime image from ``requirements-all.txt``. +- ``requirements.txt`` is what the README and runbooks tell people to install. + +Nothing kept them in step, and they drifted: both sat two jvspatial releases +behind, and both were missing four core dependencies outright (aiohttp, pymupdf, +packaging, mcp), so ``pip install -r requirements.txt`` produced an install that +could not load several actions. The failure is silent at install time and only +shows up as an ImportError deep in a run. +""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Dict, List + +REPO_ROOT = Path(__file__).resolve().parent.parent +PYPROJECT = REPO_ROOT / "pyproject.toml" +REQUIREMENTS = ("requirements.txt", "requirements-all.txt") + +_REQUIREMENT_RE = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*(?:\[[^\]]+\])?)\s*(.*)$") + + +def _canonical(name: str) -> str: + """PEP 503 normalization, minus any extras marker.""" + return re.sub(r"[-_.]+", "-", name.split("[")[0]).lower() + + +def _core_dependencies() -> Dict[str, str]: + """``{canonical_name: full_spec}`` from ``[project] dependencies``.""" + text = PYPROJECT.read_text(encoding="utf-8") + block = re.search(r"^dependencies = \[(.*?)^\]", text, re.S | re.M) + assert block, "could not locate [project] dependencies in pyproject.toml" + deps: Dict[str, str] = {} + for raw in re.findall(r'"([^"]+)"', block.group(1)): + match = _REQUIREMENT_RE.match(raw) + assert match, f"unparsable dependency spec: {raw!r}" + deps[_canonical(match.group(1))] = raw.strip() + return deps + + +def _listed(filename: str) -> Dict[str, str]: + """``{canonical_name: full_spec}`` for one requirements file.""" + listed: Dict[str, str] = {} + for line in (REPO_ROOT / filename).read_text(encoding="utf-8").splitlines(): + line = line.split("#")[0].strip() + if not line or line.startswith("-"): + continue + match = _REQUIREMENT_RE.match(line) + if match: + listed[_canonical(match.group(1))] = line + return listed + + +def test_core_dependencies_are_listed() -> None: + """Every runtime dependency must appear in both requirements files.""" + core = _core_dependencies() + missing: List[str] = [] + for filename in REQUIREMENTS: + listed = _listed(filename) + for name in core: + if name not in listed: + missing.append(f"{filename}: {core[name]}") + assert not missing, "missing from requirements files:\n " + "\n ".join(missing) + + +def test_core_dependency_specs_match() -> None: + """A listed dependency must carry the same version spec as pyproject. + + A requirements file pinning an older jvspatial than pyproject declares is + how the Docker image and the tested tree end up on different versions. + """ + core = _core_dependencies() + mismatched: List[str] = [] + for filename in REQUIREMENTS: + for name, spec in _listed(filename).items(): + expected = core.get(name) + if expected is not None and spec != expected: + mismatched.append(f"{filename}: {spec!r} != pyproject {expected!r}") + assert not mismatched, "version specs out of sync:\n " + "\n ".join(mismatched) From 85316a500faab1037092162897d4a73f093a5733 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Fri, 7 Aug 2026 11:01:42 -0400 Subject: [PATCH 2/3] fix(deps): add the action dependencies requirements-all.txt was missing The file's header promises "all core dependencies plus all optional dependencies from action info.yaml files". Across 57 action info.yaml files and their per-action requirements.txt, 22 distinct pip dependencies are declared; this file carried 3. Since Dockerfile.base builds the runtime image from it, the shipped image had no dependencies for Deepgram STT, ElevenLabs TTS, every Google action, Microsoft Excel, SerpAPI search, web_fetch, or PageIndex's LLM stack. Each fails as an ImportError when the action loads, not at install. Add the 13 that are unambiguous, taking the highest floor where actions declare different ones -- which is what pip resolves to anyway. Verified the file still resolves with a dry-run install. tiktoken is deliberately left out, and that is the finding worth reading: jvagent/pageindex declares tiktoken>=0.11.0 while Dockerfile.base installs 'tiktoken<0.8.0'. Those cannot both hold. Listing it here would let this file quietly override the Dockerfile's cap during the image build, so the contradiction wants resolving first -- by moving the cap or the floor, whichever is right. It sits in an allowlist in the test with that reason attached. Extend the guard accordingly: every action-declared pip dep must appear in requirements-all.txt, and an allowlist entry must still correspond to a real declaration, so tiktoken cannot linger after the fix. Both were mutation-checked -- dropping elevenlabs fails and names the declaring info.yaml; a bogus allowlist entry fails too. Not addressed here, reported instead: several packages are declared with different floors across actions (httpx has four variants, pyyaml three spellings). pip takes the highest, so these mislead rather than break, and harmonising them means editing ~20 info.yaml files. --- requirements-all.txt | 41 ++++++++++++++++++- tests/test_requirements_sync.py | 70 +++++++++++++++++++++++++++++++++ 2 files changed, 110 insertions(+), 1 deletion(-) diff --git a/requirements-all.txt b/requirements-all.txt index 9383727d..36d2cd4c 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -15,7 +15,11 @@ pymupdf>=1.24.0 packaging>=21.0 mcp>=1.0.0 -# Dependencies from action info.yaml files +# Dependencies from action info.yaml files. +# Every pip dep an action declares must appear here — enforced by +# tests/test_requirements_sync.py. Where actions declare different floors for +# the same package, the highest wins (that is what pip resolves to anyway). + # From jvagent/typesense_vectorstore typesense>=2.0.0 @@ -25,3 +29,38 @@ openai>=1.0.0 # Additional dependencies for email validation support # pydantic[email] provides email-validator for EmailStr type support pydantic[email]>=2.13.4 + +# From jvagent/web_fetch +beautifulsoup4 +markdownify + +# From jvagent/stt_action/deepgram and jvagent/tts_action/elevenlabs +deepgram-sdk>=6.0.0 +elevenlabs>=1.13.0 + +# From jvagent/whatsapp +filetype>=1.2.0 + +# From the jvagent/google/* actions and jvagent/pageindex_google_drive_sync_action +google-api-python-client>=2.192.0 +google-auth-httplib2>=0.3.0 +google-auth-oauthlib>=1.3.0 + +# From jvagent/web_search/serpapi +google-search-results>=2.4.2 + +# From jvagent/pageindex/pageindex_action +litellm>=1.82.0 +pypdf>=4.0.0 + +# From jvagent/microsoft/microsoft_excel_action +openpyxl>=3.1.0 + +# From jvagent/facebook_action +requests>=2.28.0 + +# NOTE: tiktoken is deliberately absent. jvagent/pageindex declares +# tiktoken>=0.11.0 while Dockerfile.base installs 'tiktoken<0.8.0' — the two +# cannot both hold. Listing it here would let this file silently override the +# Dockerfile's cap during the image build. Resolve the contradiction first, +# then add it (and drop it from the allowlist in tests/test_requirements_sync.py). diff --git a/tests/test_requirements_sync.py b/tests/test_requirements_sync.py index 6d69df5b..dbd55802 100644 --- a/tests/test_requirements_sync.py +++ b/tests/test_requirements_sync.py @@ -20,10 +20,22 @@ from pathlib import Path from typing import Dict, List +import pytest +import yaml + REPO_ROOT = Path(__file__).resolve().parent.parent PYPROJECT = REPO_ROOT / "pyproject.toml" REQUIREMENTS = ("requirements.txt", "requirements-all.txt") +# Packages an action declares that requirements-all.txt deliberately omits. +# Each needs a reason; an empty allowlist is the goal. +ACTION_DEP_EXCEPTIONS = { + # jvagent/pageindex declares tiktoken>=0.11.0 while Dockerfile.base installs + # 'tiktoken<0.8.0'. Listing it would silently override the Dockerfile cap + # during the image build, so the contradiction is resolved first. + "tiktoken", +} + _REQUIREMENT_RE = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*(?:\[[^\]]+\])?)\s*(.*)$") @@ -70,6 +82,64 @@ def test_core_dependencies_are_listed() -> None: assert not missing, "missing from requirements files:\n " + "\n ".join(missing) +def _action_pip_dependencies() -> Dict[str, List[str]]: + """``{canonical_name: [declaring info.yaml, ...]}`` across every action.""" + declared: Dict[str, List[str]] = {} + for info in sorted(REPO_ROOT.glob("jvagent/action/**/info.yaml")): + try: + data = yaml.safe_load(info.read_text(encoding="utf-8")) or {} + except yaml.YAMLError: # pragma: no cover - malformed yaml is its own bug + continue + package = data.get("package") + if not isinstance(package, dict): + continue + deps = package.get("dependencies") + pips = deps.get("pip") if isinstance(deps, dict) else None + if isinstance(pips, dict): + pips = [f"{k}{v}" for k, v in pips.items()] + for spec in pips or []: + if not isinstance(spec, str) or not spec.strip(): + continue + match = _REQUIREMENT_RE.match(spec.strip()) + if match: + name = _canonical(match.group(1)) + declared.setdefault(name, []).append(str(info.relative_to(REPO_ROOT))) + return declared + + +def test_action_dependencies_are_in_requirements_all() -> None: + """requirements-all.txt must carry every pip dep an action declares. + + Its own header promises "all core dependencies plus all optional + dependencies from action info.yaml files", and Dockerfile.base builds the + runtime image from it — so anything missing is an action that cannot run in + the shipped image. + """ + declared = _action_pip_dependencies() + assert declared, "found no action pip dependencies — parser is broken" + listed = _listed("requirements-all.txt") + missing = { + name: sources + for name, sources in declared.items() + if name not in listed and name not in ACTION_DEP_EXCEPTIONS + } + detail = "\n ".join( + f"{name} (declared by {sources[0]}" + + (f" +{len(sources) - 1} more)" if len(sources) > 1 else ")") + for name, sources in sorted(missing.items()) + ) + assert not missing, "action deps missing from requirements-all.txt:\n " + detail + + +@pytest.mark.parametrize("name", sorted(ACTION_DEP_EXCEPTIONS)) +def test_exceptions_are_still_declared_somewhere(name: str) -> None: + """Keep the allowlist honest — drop entries once the action stops needing them.""" + assert name in _action_pip_dependencies(), ( + f"{name} is allowlisted in ACTION_DEP_EXCEPTIONS but no action declares " + "it any more; remove the exception." + ) + + def test_core_dependency_specs_match() -> None: """A listed dependency must carry the same version spec as pyproject. From 8e704efea50d21c2efefe53985330d79981bb7b5 Mon Sep 17 00:00:00 2001 From: Eldon Marks Date: Fri, 7 Aug 2026 11:38:30 -0400 Subject: [PATCH 3/3] fix(docker): raise the tiktoken cap, which no longer held Dockerfile.base pinned 'tiktoken<0.8.0' while jvagent/pageindex declares tiktoken>=0.11.0. The cap is the wrong side of that, for three reasons. litellm -- declared by pageindex, and installed into the same image via requirements-all.txt -- requires tiktoken>=0.8.0,<1.0. So the cap was already unsatisfiable against the image's own dependency set; pageindex's floor only made it visible. The cap is not protecting the build either. cp312 manylinux x86_64 wheels exist at 0.7, 0.11 and 0.13, and the image is lambda/python:3.12 with --only-binary=:all:, so nothing forced a sub-0.8 pin. It dates to March and went stale. And jvagent's own use is version-agnostic: response/chunking.py and model/utils/token_estimation.py call get_encoding() and encoding_for_model(), stable across this whole range, both behind optional imports. Raise it to >=0.11.0,<1.0 -- pageindex's floor, litellm's ceiling. Verified the two co-resolve rather than assuming it. With the contradiction gone, tiktoken joins requirements-all.txt and ACTION_DEP_EXCEPTIONS empties, so the guard now covers all 22 declared action dependencies with no carve-outs. Not verified: an actual docker build of the image. The resolve is proven; the build is not. --- Dockerfile.base | 6 +++++- requirements-all.txt | 7 ++----- tests/test_requirements_sync.py | 9 ++------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/Dockerfile.base b/Dockerfile.base index e1e1228b..b827bf35 100644 --- a/Dockerfile.base +++ b/Dockerfile.base @@ -24,7 +24,11 @@ RUN set -eux && \ /var/lang/bin/python3.12 -m venv /opt/venv && \ /opt/venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel && \ /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'numpy>=1.26.0,<2.0' && \ - /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'tiktoken<0.8.0' && \ + # litellm (pulled in by jvagent/pageindex) requires tiktoken>=0.8.0,<1.0, and + # pageindex itself declares >=0.11.0 — the old 'tiktoken<0.8.0' cap could not + # satisfy either. cp312 manylinux wheels exist across this range, so + # --only-binary still resolves. + /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'tiktoken>=0.11.0,<1.0' && \ /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'pydantic[email]' && \ # /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'docling>=2.0.0' && \ # /opt/venv/bin/pip install --no-cache-dir --only-binary=:all: 'tabulate>=0.9.0' && \ diff --git a/requirements-all.txt b/requirements-all.txt index 36d2cd4c..e6402c4e 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -59,8 +59,5 @@ openpyxl>=3.1.0 # From jvagent/facebook_action requests>=2.28.0 -# NOTE: tiktoken is deliberately absent. jvagent/pageindex declares -# tiktoken>=0.11.0 while Dockerfile.base installs 'tiktoken<0.8.0' — the two -# cannot both hold. Listing it here would let this file silently override the -# Dockerfile's cap during the image build. Resolve the contradiction first, -# then add it (and drop it from the allowlist in tests/test_requirements_sync.py). +# From jvagent/pageindex (also required by litellm: >=0.8.0,<1.0) +tiktoken>=0.11.0 diff --git a/tests/test_requirements_sync.py b/tests/test_requirements_sync.py index dbd55802..87e99b73 100644 --- a/tests/test_requirements_sync.py +++ b/tests/test_requirements_sync.py @@ -28,13 +28,8 @@ REQUIREMENTS = ("requirements.txt", "requirements-all.txt") # Packages an action declares that requirements-all.txt deliberately omits. -# Each needs a reason; an empty allowlist is the goal. -ACTION_DEP_EXCEPTIONS = { - # jvagent/pageindex declares tiktoken>=0.11.0 while Dockerfile.base installs - # 'tiktoken<0.8.0'. Listing it would silently override the Dockerfile cap - # during the image build, so the contradiction is resolved first. - "tiktoken", -} +# Each needs a reason; empty is the goal, and it is currently empty. +ACTION_DEP_EXCEPTIONS: set = set() _REQUIREMENT_RE = re.compile(r"^\s*([A-Za-z0-9][A-Za-z0-9._-]*(?:\[[^\]]+\])?)\s*(.*)$")