feat: BYO engine-adapter packages & deterministic python-node containers

.github/workflows/ci.yml

@@ -19,9 +19,9 @@ jobs:
       - name: Types (contract)
         run: uv run mypy contract/continuo_engine_contract
       - name: Types (postgres adapter)
-        run: uv run --package continuo-python-runtime-postgres mypy adapters/postgres/continuo_python_runtime_postgres
+        run: uv run --package continuo-postgres-adapter mypy adapters/postgres/continuo_postgres_adapter
       - name: Types (trino adapter)
-        run: uv run --package continuo-python-runtime-trino mypy adapters/trino/continuo_python_runtime_trino
+        run: uv run --package continuo-trino-adapter mypy adapters/trino/continuo_trino_adapter
       # `-m "not image"` deselects tests/test_image_smoke_validation.py, which
       # needs a built engine image and the env naming it. Those tests run in
       # images.yml's smoke jobs, where an image actually exists. `and not

.github/workflows/images.yml

@@ -21,8 +21,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
+      - uses: astral-sh/setup-uv@v7
+      - name: Build wheels into a wheelhouse
+        run: uv build --all-packages -o wheelhouse
       - name: Build ${{ matrix.engine }} image
-        run: docker build -f Dockerfile.${{ matrix.engine }} -t cpr-smoke-${{ matrix.engine }} .
+        run: docker build -f Dockerfile.${{ matrix.engine }} --build-arg WHEEL_SOURCE=wheelhouse -t cpr-smoke-${{ matrix.engine }} .
       - name: Save image
         run: docker save cpr-smoke-${{ matrix.engine }} -o /tmp/cpr-smoke-${{ matrix.engine }}.tar
       - uses: actions/upload-artifact@v7
@@ -153,21 +156,31 @@ jobs:
         if: always()
         run: docker compose -f tests/smoke/trino-stack/docker-compose.yml down -v
 
-  # On tag push: build both engine images for linux/amd64 and linux/arm64 and
-  # push them to ghcr.io/<owner>/continuo-python-runtime-<engine>:<tag>. The
-  # engine belongs in the image NAME, not the tag: Continuo's Helm chart pins an
-  # image as `<name>:vX.Y.Z@sha256:<digest>`, so the tag must be the bare
+  # Runs on this same tag push, after the smoke jobs above pass. It builds both
+  # engine images for linux/amd64 and linux/arm64 installing the pinned
+  # versions FROM PyPI (WHEEL_SOURCE=pypi, the Dockerfile default), then
+  # pushes them to ghcr.io/<owner>/continuo-python-runtime-<engine>:<tag>. The
+  # engine belongs in the image NAME, not the tag: Continuo's Helm chart pins
+  # an image as `<name>:vX.Y.Z@sha256:<digest>`, so the tag must be the bare
   # version. Both platforms are required: domain repos build their own images
   # FROM these, and Continuo's e2e runs them in a kind cluster on aarch64
-  # developer machines as well as on amd64 CI runners, so a single-arch publish
-  # is unusable off amd64. The smoke jobs above stay single-arch — they load a
-  # `docker save` tarball, which has no multi-platform equivalent.
-  # continuo-engine-contract and the two engine adapters are installed from the
-  # build context (see Dockerfile.postgres / Dockerfile.trino) — nothing pending
-  # here.
+  # developer machines as well as on amd64 CI runners, so a single-arch
+  # publish is unusable off amd64. The smoke jobs above stay single-arch —
+  # they load a `docker save` tarball, which has no multi-platform
+  # equivalent.
+  #
+  # publish-pypi.yml runs in parallel on this same tag push, not before this
+  # job: there is no cross-workflow `needs:`, so this job cannot wait on that
+  # workflow directly. Instead the wait-until-installable step below polls
+  # PyPI until the pinned versions this tag just published are actually
+  # installable, which is what orders this build after publish-pypi.yml
+  # completes without a cross-workflow dependency.
+  # -test tags publish to TestPyPI only (see publish-pypi.yml), so this job
+  # skips them: there is no real-PyPI installable version to build an image
+  # from for those.
   publish:
-    if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-test')
     needs: [smoke-postgres, smoke-trino]
+    if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-test')
     strategy:
       matrix:
         engine: [postgres, trino]
@@ -185,6 +198,26 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+      # The pinned distributions declare Requires-Python >= 3.14, so the
+      # availability probe must resolve them under a 3.14 interpreter — the
+      # runner's default python is older and pip would reject them even after
+      # they are published, spinning out every retry. (The image build itself
+      # installs inside python:3.14-slim, so it is unaffected; this is only for
+      # the host-side probe below.)
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.14'
+      - name: Wait until the pinned versions are installable from PyPI
+        run: |
+          set -eu
+          req="image-requirements-${{ matrix.engine }}.txt"
+          for i in $(seq 1 30); do
+            if python -m pip download --dest /tmp/probe -r "$req" >/tmp/probe.log 2>&1; then
+              echo "installable"; exit 0
+            fi
+            echo "attempt $i: not yet on index; sleeping"; sleep 20
+          done
+          echo "pinned versions never became installable"; cat /tmp/probe.log; exit 1
       - name: Build and push ${{ matrix.engine }} image
         uses: docker/build-push-action@v7
         with:

.github/workflows/publish-pypi.yml

@@ -1,28 +1,26 @@
 name: publish-pypi
 
-# Publishes both PyPI distributions this repo owns — continuo-python-runtime
-# (the harness) and continuo-engine-contract (the port, result-block format,
-# and shared guards) — via PyPI Trusted Publishing (OIDC, no stored token).
-#
-# The two engine adapters (continuo-python-runtime-postgres /
-# continuo-python-runtime-trino) are deliberately NOT published here. Nothing
-# installs them from an index: the engine images build them from the build
-# context (see Dockerfile.postgres / Dockerfile.trino and images.yml), and the
-# harness package does not depend on them — adapters are found at run time
-# through the `continuo_engine.adapters` entry-point group, inside the image
-# that installed exactly one. Publishing them would only add releases with no
-# consumer. They remain uv workspace members and are still built, typed, and
-# tested by ci.yml.
+# Publishes all four PyPI distributions this repo owns — continuo-python-runtime
+# (the harness), continuo-engine-contract (the port, result-block format, and
+# shared guards), and the two engine adapters (continuo-postgres-adapter,
+# continuo-trino-adapter) — via PyPI Trusted Publishing (OIDC, no stored
+# token). They publish together on a single v* tag, each at its own
+# pyproject version, so a tag that only bumps the runtime still carries an
+# unchanged adapter version along for the ride; skip-existing (below) makes
+# that a no-op rather than a duplicate-upload failure. A release-time guard
+# (scripts/check_version_bumps.py) refuses a tag that changed a package's
+# source without bumping its version, so skip-existing can never silently
+# ship stale adapter code under an unchanged version number.
 #
 # Tag glob note: `v*` is the repository's single release pattern — the same tag
 # that images.yml builds and pushes both engine images from, so one tag ships
 # the whole release. GitHub Actions tag globs anchor at character 1, so `v*`
 # claims every tag beginning with "v"; no other pattern may be introduced.
 # Tag `v<ver>-test<n>` publishes to TestPyPI; `v<ver>` publishes to real PyPI.
 # The GitHub environment name is what the PyPI "pending publisher" is
-# registered against — both project names need one on each index.
+# registered against — all four project names need one on each index.
 #
-# Both distributions are built into a single `dist/` and uploaded in one
+# All four distributions are built into a single `dist/` and uploaded in one
 # publish call, so there is no ordering constraint between them. The runtime
 # wheel declares continuo-engine-contract as a dependency and resolves it from
 # the index at install time ([tool.uv.sources] is dev-only and is not embedded
@@ -44,17 +42,42 @@ jobs:
       contents: read # actions/checkout needs read access to the repo
     steps:
       - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
       - uses: astral-sh/setup-uv@v7
+      - name: Refuse a tag that changed a package without bumping its version
+        run: python scripts/check_version_bumps.py
       - name: Test before publishing
-        # `-m "not image"` deselects the image smoke tests; images.yml runs
-        # those against a built engine image.
+        # Gate all four published projects, including both adapter suites — an
+        # adapter regression must block its own immutable PyPI upload, and
+        # images.yml runs in parallel so its smoke failure cannot. Each suite is
+        # a SEPARATE pytest invocation: every workspace member's tests/ is its
+        # own `tests` package (with __init__.py + conftest.py), so running two
+        # of them in one process collides on the `tests.conftest` module name
+        # (see the [tool.pytest.ini_options] note in pyproject.toml). `not image
+        # and not integration` deselects the tests needing a built engine image
+        # or a live warehouse (images.yml / the smoke jobs cover those). The
+        # step's default `set -e` fails the gate if any invocation fails.
         run: |
           uv sync --all-packages --all-groups
-          uv run pytest tests contract/tests -m "not image" -q
+          uv run pytest tests contract/tests -m "not image and not integration" -q
+          uv run pytest adapters/postgres/tests -m "not image and not integration" -q
+          uv run pytest adapters/trino/tests -m "not image and not integration" -q
       - name: Build the contract sdist + wheel
         run: uv build --package continuo-engine-contract -o dist
       - name: Build the runtime sdist + wheel
         run: uv build -o dist
+      - name: Build the postgres adapter sdist + wheel
+        run: uv build --package continuo-postgres-adapter -o dist
+      - name: Build the trino adapter sdist + wheel
+        run: uv build --package continuo-trino-adapter -o dist
+      # Adapters change rarely, so most v* tags carry an unchanged adapter
+      # version alongside a bumped runtime/contract version. Without
+      # skip-existing, re-uploading that unchanged version 400s and fails the
+      # whole publish. Trade-off: a changed package published without a
+      # version bump is then silently skipped too — the
+      # check_version_bumps.py guard above is what catches that case, before
+      # this job's build/publish steps ever run.
       - name: Publish to TestPyPI
         if: contains(github.ref_name, '-test')
         # Pinned to a commit SHA. release/v1 is a moving branch, and this job
@@ -64,6 +87,7 @@ jobs:
         with:
           repository-url: https://test.pypi.org/legacy/
           packages-dir: dist
+          skip-existing: true
       - name: Publish to PyPI
         if: ${{ !contains(github.ref_name, '-test') }}
         # Pinned to a commit SHA. release/v1 is a moving branch, and this job
@@ -72,3 +96,4 @@ jobs:
         uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1
         with:
           packages-dir: dist
+          skip-existing: true

CONTRIBUTING.md

@@ -57,8 +57,8 @@ uv run ruff check .
 uv run ruff check contract
 uv run mypy continuo_python_runtime
 uv run mypy contract/continuo_engine_contract
-uv run --package continuo-python-runtime-postgres mypy adapters/postgres/continuo_python_runtime_postgres
-uv run --package continuo-python-runtime-trino mypy adapters/trino/continuo_python_runtime_trino
+uv run --package continuo-postgres-adapter mypy adapters/postgres/continuo_postgres_adapter
+uv run --package continuo-trino-adapter mypy adapters/trino/continuo_trino_adapter
 uv run pytest --cov=continuo_python_runtime -m "not image and not integration" -v
 uv run pytest tests/test_csv_readers_integration.py tests/test_validation_runner.py -m integration -v
 uv run pytest contract/tests -v

Dockerfile.postgres

@@ -1,16 +1,50 @@
 FROM python:3.14-slim
-# One dual-role image per engine: the python-node runtime harness (default
-# command `run`, domain repos build FROM this) and the blue/green validation
-# runner (`validation-op`, continuo's executor sets it as the container
-# command). Everything installs from this repo's own workspace paths in one
-# pip transaction — no PyPI propagation lag; images always build from repo head.
-#   docker build -f Dockerfile.postgres .
-COPY pyproject.toml README.md /src/
-COPY contract /src/contract
-COPY continuo_python_runtime /src/continuo_python_runtime
-COPY adapters/postgres /src/adapters/postgres
-RUN pip install --no-cache-dir /src/contract /src/adapters/postgres /src \
-    && rm -rf /src
+# One dual-role image per engine — the python-node runtime harness (default
+# command `run`) and the blue/green validation runner (`validation-op`). It
+# installs the PUBLISHED, versioned libraries, never the repo source:
+#   WHEEL_SOURCE=pypi       (release) installs the pins from PyPI. PyPI versions
+#                           are immutable and the image is consumed
+#                           digest-pinned, so no hash lock is needed here.
+#   WHEEL_SOURCE=wheelhouse (CI/PR)   installs the same pins from ./wheelhouse
+#                           (built by CI) so an unreleased change is testable.
+# The wheelhouse holds only this repo's own first-party wheels (built via
+# `uv build --all-packages`); third-party dependencies always resolve from
+# PyPI in both branches, so the wheelhouse branch never passes --no-index to
+# the requirements install below.
+# The pinned versions live in image-requirements-postgres.txt, kept equal to the
+# repo's pyproject versions by tests/test_image_requirements_sync.py.
+#
+# The wheelhouse branch installs its three first-party wheels (contract,
+# runtime, adapter) by exact local file first, with --no-index --no-deps: a
+# plain `--find-links=/tmp/wheelhouse -r req.txt` is not enough, because pip
+# does not prefer a find-links wheel over an index match at the same
+# name+version, and continuo-python-runtime / continuo-engine-contract are
+# already published on PyPI under the version this repo's HEAD currently
+# carries (a routine dependency bump lands between releases without a version
+# bump). Left unpinned-by-file, pip silently installs the last PyPI release's
+# transitive pins instead of the wheel this build just produced, defeating
+# the point of testing an unreleased change. Installing the exact local files
+# first (no index contacted at all for this step) is unambiguous; the second
+# install then only has PyPI-only third-party deps left to resolve, and finds
+# the three first-party packages already satisfied.
+ARG WHEEL_SOURCE=pypi
+COPY image-requirements-postgres.txt /tmp/req.txt
+# Optional COPY: the bracket glob matches ./wheelhouse when it exists (the CI/PR
+# WHEEL_SOURCE=wheelhouse path builds it) and no-ops when it is absent, so a
+# release build or a plain `docker build -f Dockerfile.postgres .` needs no
+# pre-created directory. Only the wheelhouse branch below reads its contents.
+COPY wheelhous[e] /tmp/wheelhouse
+RUN set -eu; \
+    if [ "$WHEEL_SOURCE" = "wheelhouse" ]; then \
+      pip install --no-cache-dir --no-index --no-deps \
+        /tmp/wheelhouse/continuo_engine_contract-*.whl \
+        /tmp/wheelhouse/continuo_python_runtime-*.whl \
+        /tmp/wheelhouse/continuo_postgres_adapter-*.whl; \
+      pip install --no-cache-dir -r /tmp/req.txt; \
+    else \
+      pip install --no-cache-dir -r /tmp/req.txt; \
+    fi; \
+    rm -rf /tmp/req.txt /tmp/wheelhouse
 # PYTHONPATH is belt-and-braces: the harness also inserts APP_ROOT and the
 # node script's own directory at the front of sys.path before executing it.
 ENV CONTRACT_DIR=/app/contracts APP_ROOT=/app PYTHONPATH=/app

Dockerfile.trino

@@ -1,16 +1,50 @@
 FROM python:3.14-slim
-# One dual-role image per engine: the python-node runtime harness (default
-# command `run`, domain repos build FROM this) and the blue/green validation
-# runner (`validation-op`, continuo's executor sets it as the container
-# command). Everything installs from this repo's own workspace paths in one
-# pip transaction — no PyPI propagation lag; images always build from repo head.
-#   docker build -f Dockerfile.trino .
-COPY pyproject.toml README.md /src/
-COPY contract /src/contract
-COPY continuo_python_runtime /src/continuo_python_runtime
-COPY adapters/trino /src/adapters/trino
-RUN pip install --no-cache-dir /src/contract /src/adapters/trino /src \
-    && rm -rf /src
+# One dual-role image per engine — the python-node runtime harness (default
+# command `run`) and the blue/green validation runner (`validation-op`). It
+# installs the PUBLISHED, versioned libraries, never the repo source:
+#   WHEEL_SOURCE=pypi       (release) installs the pins from PyPI. PyPI versions
+#                           are immutable and the image is consumed
+#                           digest-pinned, so no hash lock is needed here.
+#   WHEEL_SOURCE=wheelhouse (CI/PR)   installs the same pins from ./wheelhouse
+#                           (built by CI) so an unreleased change is testable.
+# The wheelhouse holds only this repo's own first-party wheels (built via
+# `uv build --all-packages`); third-party dependencies always resolve from
+# PyPI in both branches, so the wheelhouse branch never passes --no-index to
+# the requirements install below.
+# The pinned versions live in image-requirements-trino.txt, kept equal to the
+# repo's pyproject versions by tests/test_image_requirements_sync.py.
+#
+# The wheelhouse branch installs its three first-party wheels (contract,
+# runtime, adapter) by exact local file first, with --no-index --no-deps: a
+# plain `--find-links=/tmp/wheelhouse -r req.txt` is not enough, because pip
+# does not prefer a find-links wheel over an index match at the same
+# name+version, and continuo-python-runtime / continuo-engine-contract are
+# already published on PyPI under the version this repo's HEAD currently
+# carries (a routine dependency bump lands between releases without a version
+# bump). Left unpinned-by-file, pip silently installs the last PyPI release's
+# transitive pins instead of the wheel this build just produced, defeating
+# the point of testing an unreleased change. Installing the exact local files
+# first (no index contacted at all for this step) is unambiguous; the second
+# install then only has PyPI-only third-party deps left to resolve, and finds
+# the three first-party packages already satisfied.
+ARG WHEEL_SOURCE=pypi
+COPY image-requirements-trino.txt /tmp/req.txt
+# Optional COPY: the bracket glob matches ./wheelhouse when it exists (the CI/PR
+# WHEEL_SOURCE=wheelhouse path builds it) and no-ops when it is absent, so a
+# release build or a plain `docker build -f Dockerfile.trino .` needs no
+# pre-created directory. Only the wheelhouse branch below reads its contents.
+COPY wheelhous[e] /tmp/wheelhouse
+RUN set -eu; \
+    if [ "$WHEEL_SOURCE" = "wheelhouse" ]; then \
+      pip install --no-cache-dir --no-index --no-deps \
+        /tmp/wheelhouse/continuo_engine_contract-*.whl \
+        /tmp/wheelhouse/continuo_python_runtime-*.whl \
+        /tmp/wheelhouse/continuo_trino_adapter-*.whl; \
+      pip install --no-cache-dir -r /tmp/req.txt; \
+    else \
+      pip install --no-cache-dir -r /tmp/req.txt; \
+    fi; \
+    rm -rf /tmp/req.txt /tmp/wheelhouse
 # PYTHONPATH is belt-and-braces: the harness also inserts APP_ROOT and the
 # node script's own directory at the front of sys.path before executing it.
 ENV CONTRACT_DIR=/app/contracts APP_ROOT=/app PYTHONPATH=/app