From d20daa7c49ec482dcdc9e05ff3500ff7a31d5de5 Mon Sep 17 00:00:00 2001 From: harshsinghhsr Date: Sun, 6 Sep 2026 15:35:06 +0530 Subject: [PATCH 1/2] test: run the integration suite locally against an AWS emulator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The end-to-end path was gated behind DYNAVEC_LIVE=1 and a real account, so in practice it never ran. It now also accepts AWS_ENDPOINT_URL, and `docker compose up -d --wait` starts floci — currently the only emulator implementing s3vectors. boto3 reads AWS_ENDPOINT_URL natively, so no production code changed. Running the suite for the first time exposed a bug in the test itself: it searched the "default" namespace while upserting to "it", so it could never pass, on the emulator or on real AWS. Also drops moto[dynamodb] from dev deps; nothing imports it. --- .github/workflows/ci.yml | 23 ++++++++++++++ README.md | 30 ++++++++++++++++++ docker-compose.yml | 25 +++++++++++++++ pyproject.toml | 1 - tests/integration/test_live_aws.py | 49 ++++++++++++++++++++---------- 5 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 docker-compose.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d10d15b..7c926b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,3 +24,26 @@ jobs: run: uv run --no-sync ruff check src benchmarks - name: Test run: uv run --no-sync pytest -q + + # Runs the real end-to-end path (provisioning -> upsert -> query -> hydrate) + # against a local AWS emulator. floci is used because it is currently the only + # emulator implementing s3vectors; LocalStack has it in backlog only. + integration-local: + runs-on: ubuntu-latest + env: + AWS_ENDPOINT_URL: http://localhost:4566 + steps: + - uses: actions/checkout@v4 + - name: Start the local AWS emulator + run: docker compose up -d --wait # --wait blocks on the compose healthcheck + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + python-version: "3.12" + - name: Install (with dev extras) + run: uv pip install -e ".[dev]" + - name: Integration test + run: uv run --no-sync pytest tests/integration -v + - name: Emulator logs + if: failure() + run: docker compose logs floci diff --git a/README.md b/README.md index 0506c7b..c213d72 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,36 @@ python -m benchmarks.run_benchmark --backend dynavec \ **v0.2** — everything in the table above, on top of the v0.1 hybrid core (pluggable embedders, RRF, MMR, provisioning). 61 tests. **Roadmap (v0.3):** native asyncio client (`aioboto3`), in-process `hnswlib` hot tier, sparse/BM25 hybrid computed from DynamoDB, sort-key graph adjacency for very high fan-out, and turnkey file parsers (PDF/DOCX/PPTX/XLSX) as ingestion sources. +## Local development (no AWS account) + +The unit suite needs nothing — `pytest -q` runs fully offline against in-memory fakes. +To exercise the *real* path (provisioning → `s3vectors` → DynamoDB → hydration) without +an AWS bill, run a local emulator. Use [floci](https://github.com/floci-io/floci): it is +currently the only one that implements `s3vectors` (LocalStack has it in backlog only). + +```bash +docker compose up -d --wait # starts floci, waits until healthy +export AWS_ENDPOINT_URL=http://localhost:4566 # the only variable you need +pytest tests/integration -v +``` + +`docker compose down` when you're done; storage is in-memory, so every restart gives +you a clean account back. + +That is the whole setup — **dynavec needs no code change or endpoint config**. boto3 +reads `AWS_ENDPOINT_URL` natively, so every `session.client(...)` in `stores/` and +`provisioning.py` points at the emulator on its own; the test module fills in dummy +credentials when it sees that variable. The same variable makes the `examples/` +scripts run locally (swap the embedder for a fake one, or set a real `OPENAI_API_KEY` +— embedders call their provider, not AWS). + +CI runs this as the `integration-local` job on every push. It catches wiring bugs the +fakes can't — wrong request shape, missing pagination, a provisioning call that never +fires. It is **not** a substitute for real AWS: an emulator's cosine scan is not S3 +Vectors' ANN index and has none of its eventual-consistency behaviour, so keep +`DYNAVEC_LIVE=1` (see [`tests/integration/test_live_aws.py`](tests/integration/test_live_aws.py)) +as the pre-release gate. + ## Publishing (maintainers) `dynavec` publishes to **PyPI**; both `pip` and `uv` install from there (there is no separate "uv registry"). diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5ac6793 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +# Local AWS for dynavec development — no account, no cost. +# +# docker compose up -d --wait +# export AWS_ENDPOINT_URL=http://localhost:4566 +# pytest tests/integration -v +# +# floci is used because it is currently the only emulator that implements +# s3vectors; LocalStack has it in backlog only (localstack/localstack#13498). +# boto3 reads AWS_ENDPOINT_URL natively, so dynavec needs no endpoint config. +services: + floci: + image: floci/floci:latest + container_name: dynavec-floci + ports: + - "4566:4566" + environment: + FLOCI_DEFAULT_REGION: us-east-1 + # in-memory: every `down` gives you a clean account back. + FLOCI_STORAGE_MODE: memory + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:4566/_localstack/health"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 3s diff --git a/pyproject.toml b/pyproject.toml index f1583f6..6cab5a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,6 @@ all = [ dev = [ "pytest>=8.0", "pytest-cov>=5.0", - "moto[dynamodb]>=5.0", "ruff>=0.6", ] benchmark = [ diff --git a/tests/integration/test_live_aws.py b/tests/integration/test_live_aws.py index bee083c..2ce199b 100644 --- a/tests/integration/test_live_aws.py +++ b/tests/integration/test_live_aws.py @@ -1,15 +1,21 @@ -"""Live AWS integration test (opt-in). +"""End-to-end integration test (opt-in) — runs against real AWS or a local emulator. -Runs the real thing end-to-end against YOUR account: auto-provisions an S3 vector -bucket + index + DynamoDB table, upserts vectors, queries, verifies metadata -filtering + document hydration, then deletes every resource it created. +Auto-provisions an S3 vector bucket + index + DynamoDB table, upserts vectors, +queries, verifies metadata filtering + document hydration, then deletes every +resource it created. -Enable with: +Against real AWS (costs money, uses YOUR account): export DYNAVEC_LIVE=1 export AWS_REGION=us-east-1 # a region where S3 Vectors is available pytest tests/integration/test_live_aws.py -v -s -Skipped by default so the normal suite needs no AWS credentials or cost. +Against a local emulator (free, no account) — see "Local development" in the README: + docker compose up -d --wait + export AWS_ENDPOINT_URL=http://localhost:4566 + pytest tests/integration/test_live_aws.py -v -s + +boto3 honours AWS_ENDPOINT_URL natively, so dynavec needs no endpoint plumbing. +Skipped by default: the normal suite needs no credentials, no Docker, no cost. """ from __future__ import annotations @@ -21,14 +27,23 @@ import numpy as np import pytest +LOCAL_ENDPOINT = os.environ.get("AWS_ENDPOINT_URL") + +if LOCAL_ENDPOINT: + # Emulators accept any non-empty credentials, but boto3 still refuses to sign + # without them. setdefault so a real profile/role is never overridden. + os.environ.setdefault("AWS_ACCESS_KEY_ID", "test") + os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "test") + pytestmark = pytest.mark.skipif( - os.environ.get("DYNAVEC_LIVE") != "1", - reason="set DYNAVEC_LIVE=1 (and AWS creds) to run live AWS integration tests", + os.environ.get("DYNAVEC_LIVE") != "1" and not LOCAL_ENDPOINT, + reason="set DYNAVEC_LIVE=1 (real AWS) or AWS_ENDPOINT_URL (local emulator) to run", ) REGION = os.environ.get("AWS_REGION", "us-east-1") DIM = 32 N = 40 +NS = "it" # searches must target the same namespace the upserts wrote to def _unit(rng, n, d): @@ -68,9 +83,11 @@ def live_db(): db.close() -def _query_with_retry(db, vector, top_k, filter=None, tries=8, delay=3): +def _query_with_retry(db, vector, top_k, filter=None, namespace=NS, tries=8, delay=3): + if LOCAL_ENDPOINT: + delay = 0 # emulators are immediately consistent; don't burn 24s in CI for _ in range(tries): - hits = db.search(vector=vector, top_k=top_k, filter=filter) + hits = db.search(vector=vector, top_k=top_k, filter=filter, namespace=namespace) if hits: return hits time.sleep(delay) # S3 Vectors is eventually consistent after ingest @@ -91,7 +108,7 @@ def test_provision_upsert_search_roundtrip(live_db): ) for i in range(N) ] - live_db.upsert(docs, namespace="it") + live_db.upsert(docs, namespace=NS) # nearest neighbor of vec[0] should be v0 itself hits = _query_with_retry(live_db, vecs[0].tolist(), top_k=5) @@ -110,11 +127,11 @@ def test_update_and_delete(live_db): rng = np.random.default_rng(1) v = _unit(rng, 1, DIM)[0].tolist() - live_db.upsert([Document(id="u1", vector=v, text="original", metadata={"tag": "a"})], namespace="it") + live_db.upsert([Document(id="u1", vector=v, text="original", metadata={"tag": "a"})], namespace=NS) - live_db.update("u1", namespace="it", metadata={"tag": "b"}, merge_metadata=False) - got = live_db.get(["u1"], namespace="it")[0] + live_db.update("u1", namespace=NS, metadata={"tag": "b"}, merge_metadata=False) + got = live_db.get(["u1"], namespace=NS)[0] assert got.metadata["tag"] == "b" - live_db.delete(["u1"], namespace="it") - assert live_db.get(["u1"], namespace="it") == [] + live_db.delete(["u1"], namespace=NS) + assert live_db.get(["u1"], namespace=NS) == [] From 5f0261d637894365d9c60860cb681f46406e23c6 Mon Sep 17 00:00:00 2001 From: harshsinghhsr Date: Mon, 7 Sep 2026 18:48:58 +0530 Subject: [PATCH 2/2] ci: pin the emulator image, use its native health endpoint, don't block Review feedback: - pin floci/floci:2.0.1; `latest` moves as nightlies ship most days - healthcheck hits /_floci/health rather than the /_localstack/health compatibility alias - integration-local is continue-on-error until it has a track record, so a third-party image or Docker hiccup can't red-X an unrelated PR --- .github/workflows/ci.yml | 6 ++++++ docker-compose.yml | 6 ++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c926b2..7d2011d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,8 +28,14 @@ jobs: # Runs the real end-to-end path (provisioning -> upsert -> query -> hydrate) # against a local AWS emulator. floci is used because it is currently the only # emulator implementing s3vectors; LocalStack has it in backlog only. + # + # Non-blocking on purpose: this depends on a third-party image and a Docker + # daemon, so an upstream hiccup must never red-X an unrelated PR. It still runs + # on every PR, which is how it earns a track record -- drop continue-on-error + # once it has one. integration-local: runs-on: ubuntu-latest + continue-on-error: true env: AWS_ENDPOINT_URL: http://localhost:4566 steps: diff --git a/docker-compose.yml b/docker-compose.yml index 5ac6793..0f0dfe1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,7 +9,9 @@ # boto3 reads AWS_ENDPOINT_URL natively, so dynavec needs no endpoint config. services: floci: - image: floci/floci:latest + # pinned: `latest` moves (nightlies ship most days) and would make CI + # non-reproducible. Bump deliberately. + image: floci/floci:2.0.1 container_name: dynavec-floci ports: - "4566:4566" @@ -18,7 +20,7 @@ services: # in-memory: every `down` gives you a clean account back. FLOCI_STORAGE_MODE: memory healthcheck: - test: ["CMD", "curl", "-sf", "http://localhost:4566/_localstack/health"] + test: ["CMD", "curl", "-sf", "http://localhost:4566/_floci/health"] interval: 2s timeout: 3s retries: 30