diff --git a/.dockerignore b/.dockerignore index a38a99b..ea2c823 100644 --- a/.dockerignore +++ b/.dockerignore @@ -39,10 +39,6 @@ build/ coverage/ *.tsbuildinfo -# The frontend is built from its own context (apps/frontend); the backend -# images have no use for it. -apps/frontend/ - # Documentation, prototypes, and 6.6 MB of recorded evaluation runs. The # services read none of them at runtime — the figures are baked into the # frontend bundle at build time by scripts/sync_eval_artifacts.py. diff --git a/.env.production.example b/.env.production.example index b93ae99..8926777 100644 --- a/.env.production.example +++ b/.env.production.example @@ -77,6 +77,19 @@ INTERNAL_API_KEY=__SET_PROD_INTERNAL_API_KEY__ LOG_LEVEL=info +# Optional clone-target allowlist. Empty means any public host, which is what a +# demo indexing arbitrary open-source repositories needs. Set it and it becomes +# the only rule in the URL check that states what is allowed rather than what is +# obviously wrong. Subdomains match. +# REPO_URL_ALLOWED_HOSTS=github.com,gitlab.com + +# Cloned checkouts were never removed, so the volume grew with every distinct +# repository anyone submitted. 0 keeps that. Setting a cap evicts the oldest +# checkouts before a clone — an evicted repository keeps search and the call +# graph (those read Postgres) and loses read_file / grep / list_directory until +# it is re-indexed. Pick a number that fits the volume. +# WORKDIR_MAX_REPOS=8 + # ============================================================ # Retrieval models # diff --git a/Deploy.md b/Deploy.md index cd8a574..891bad4 100644 --- a/Deploy.md +++ b/Deploy.md @@ -25,7 +25,8 @@ Outstanding Work list in the same PR, not tracked in two places. is written from an approved plan, and plans outrun implementations. Verify before relying on a claim here, and correct it when it is wrong. -**Status: PR 1–4 landed. PR 5 not started, and nothing is deployed yet.** The pre-deployment state +**Status: PR 1–5 landed. Nothing is deployed yet** — every remaining item is +in § 10.2 and needs a Railway account. The pre-deployment state is frozen at tag `v1.0-submission` (`a4612b8`). Nothing is deployed anywhere yet. Both hosted model APIs have been exercised end to end from a running local stack — see § 2.1 and § 6 PR 2 for the observed values — but **no index has been @@ -496,13 +497,47 @@ The combined `agent-worker` container has **not been run**, locally or otherwise. Its entrypoint is Railway-only and the developer stack does not use it, so nothing here exercises it. -### PR 5 — `fix/audit-followups` +### PR 5 — `fix/audit-followups` — **done** Findings that are not launch blockers but should not ship unaddressed. See the register in § 7 for F-04, F-06, F-07, F-08, F-10, F-14, F-15. -**Acceptance.** Each item either fixed with a test, or explicitly deferred in -this document with a reason. Silence is not an outcome. +**Acceptance — met.** Eight items, 40 new tests, three commits grouped by what +they are about rather than by which finding they close. + +| Finding | What changed | +|---|---| +| F-03 | Request bodies had no size limit anywhere. `query` had a `min_length` and no max, and history was trimmed only *after* parsing | +| F-04 | Both the API and the agent refuse to start on the placeholder key. The length rule applies only when the deployment authenticates its users, which is a proxy for production and is labelled as one | +| F-06 | Resolution checking, credentials refused rather than stripped, and an optional allowlist. **Rebinding stays open** — see below | +| F-07 | Tool cache keyed by `index_revision`, read once per query. No revision means no cache rather than a shared key | +| F-08 | Optional oldest-first cap, off by default. **Deliberately less than "solved"** — see below | +| F-10 | `/readyz`, including the embedding-dimension check § 5.4 asks for | +| F-14 | `--` before the clone URL, and `GIT_TERMINAL_PROMPT=0` | +| F-15 | Timeout, match caps, file-size cap. **The Python fallback is not fully bounded** — see below | + +**Three limits this PR does not remove, stated because a closed finding that +quietly leaves the hole open is worse than an open one.** + +1. **DNS rebinding is not closed.** Every resolved address is checked, but git + resolves the name again when it clones, and an answer that changes in between + defeats the check. The fix is egress filtering at the network — on Railway, + that is a platform capability this project does not configure. +2. **The workdir cap is a trade, not a cleanup.** The agent reads those trees at + query time, so evicting a `ready` repository's checkout costs it `read_file`, + `grep` and `list_directory` until it is re-indexed; search and the call graph + carry on from Postgres. That is why it is off by default and why the tool's + error message now says which half stopped working. The durable answer is + § 11 item 2 — stop reading source from a filesystem. +3. **The `grep` Python fallback can still block.** Python cannot time out a + single `re` call, so catastrophic backtracking against one long line is + unbounded. Everything countable is capped. The agent image installs ripgrep, + so that path runs only where it is absent. + +Also unchanged on purpose: the Docker health checks still probe `/healthz`. +`/readyz` is for a load balancer, and wiring it into `depends_on: +condition: service_healthy` would make an unrelated dependency outage block +container startup chains locally. --- @@ -519,19 +554,19 @@ as leads to verify when the PR touches them, not as established fact. |---|---|---|---|---|---| | F-01 | ~~No authentication of any kind on `/api/v1/*`~~ **— closed in PR 3.** A shared-account session gate now covers repos, query and inspector; `/internal/*` keeps its own shared-key check. Still one principal, not a tenant model (D-6) | `apps/api/src/dcode_api/main.py` | blocker | 3 | **direct** | | F-02 | ~~The `api` service in the production Compose file has no embedding or reranker configuration~~ **— fixed in PR 1, and it was wider than this row said: `worker` and `agent` were short too. See § 6 PR 1.** | `docker-compose.prod.yml` | blocker | 1 | **direct** | -| F-03 | **Partly closed in PR 3.** Anonymous access is gone and a per-session daily query cap bounds the metered path. **Still open: no request-size limit** — `QueryRequest.query` has `min_length` and no `max_length` | `packages/shared/src/dcode_shared/schemas.py:132` | blocker → medium | 3 → 5 | audit | -| F-04 | `INTERNAL_API_KEY` defaults to a literal published in this repository, with no startup guard outside Compose's `:?` operator — and Railway sets variables per service, so that operator does not protect the deployment | `packages/shared/src/dcode_shared/settings.py:33` | high | 5 | audit | +| F-03 | ~~No rate limiting, no request-size limit, no quota~~ **— closed across PR 3 and PR 5.** Anonymous access gone, per-session daily cap, and bounds on the URL, the query, each turn and the turn count | `packages/shared/src/dcode_shared/schemas.py` | blocker | 3, 5 | **direct** | +| F-04 | ~~`INTERNAL_API_KEY` defaults to a published literal with no startup guard outside Compose~~ **— fixed in PR 5.** API and agent both refuse to boot on it | `packages/shared/src/dcode_shared/internal.py` | high | 5 | **direct** | | F-05 | ~~`/docs`, `/redoc` and `/openapi.json` are publicly served~~ **— closed in PR 3**, off in production and pinned by a test; observed 404 on a gated container | `apps/api/src/dcode_api/main.py` | high | 3 | **direct** | -| F-06 | SSRF: repository URL validation rejects literal private IPs but has no host allowlist and no resolution check, so a hostname resolving to an internal address passes | `apps/api/src/dcode_api/routes/repos.py:203-224` | high | 5 | audit | -| F-07 | The agent tool cache key omits `index_revision`, so re-indexing a repository does not invalidate cached tool results for up to 24 h | `apps/agent/src/dcode_agent/graph.py:119` | high | 5 | audit | -| F-08 | Cloned workdirs are never cleaned up, for any repository, ever. On a fixed Railway volume this is unbounded growth | `apps/worker/src/dcode_worker/stages/clone.py:17-22` | high | 5 | audit | +| F-06 | **Mostly fixed in PR 5**: resolution checked, credentials refused, optional allowlist added. **Rebinding remains open** — git re-resolves at clone time and that needs network egress filtering | `apps/api/src/dcode_api/routes/repos.py` | high | 5 | **direct** | +| F-07 | ~~The agent tool cache key omits `index_revision`~~ **— fixed in PR 5**, keyword-only with no default so a new call site cannot reintroduce it | `packages/shared/src/dcode_shared/cache.py` | high | 5 | **direct** | +| F-08 | **Partly addressed in PR 5**: an optional oldest-first cap, **off by default**, because eviction costs an evicted repository its file tools until re-index. The durable fix is § 11 item 2 | `apps/worker/src/dcode_worker/stages/prune.py` | high | 5 | **direct** | | F-09 | ~~No repository list endpoint~~ **— fixed in PR 4.** `GET /api/v1/repos`, merged behind the localStorage recents so a reader on a new device sees what is indexed. Verified live against the local database: 8 repositories returned | `apps/api/src/dcode_api/routes/repos.py` | high | 4 | **direct** | -| F-10 | `/healthz` returns `ok` unconditionally — it reports healthy with every dependency down. No readiness probe exists | `apps/api/src/dcode_api/main.py:51-54` | medium | 5 | audit | +| F-10 | ~~No readiness probe~~ **— fixed in PR 5.** `/readyz` covers database, Redis and the embedding-dimension agreement; `/healthz` stays shallow on purpose | `apps/api/src/dcode_api/readiness.py` | medium | 5 | **direct** | | F-11 | ~~The two documented production Compose invocations disagree; one of them republishes Postgres, Redis and the RabbitMQ UI~~ **— fixed in PR 1.** | `README.md`, `.env.production.example` | medium | 1 | **direct** | | F-12 | ~~`RERANKER_ENDPOINT` defaults to a dead loopback address in the production template~~ **— fixed in PR 1, pinned by a test.** | `docker-compose.prod.yml`, `.env.production.example` | medium | 1 | **direct** | | F-13 | Embedding retries are 12 attempts against a 300 s timeout; with `prefetch_count=1` one bad batch stalls the entire indexing queue. **Knob added in PR 1 (`EMBEDDING_TIMEOUT_SECONDS`); the values are set in PR 2, where the path that needs them exists.** | `packages/shared/src/dcode_shared/embedding.py:19-20` | medium | 1 → 2 | audit | -| F-14 | `git clone` receives the repository URL with no `--` separator, so a URL beginning with `-` is parsed as an option. Gateway validation is the only defence; the worker has none | `apps/worker/src/dcode_worker/stages/clone.py:22` | medium | 5 | audit | -| F-15 | The `grep` tool runs `rg` with no timeout and buffers all output in memory; the pure-Python fallback compiles a user-supplied regex with no timeout | `apps/agent/src/dcode_agent/tools/grep.py:44-58`, `:79-101` | medium | 5 | audit | +| F-14 | ~~`git clone` receives the URL with no `--` separator~~ **— fixed in PR 5**, plus `GIT_TERMINAL_PROMPT=0` | `apps/worker/src/dcode_worker/stages/clone.py` | medium | 5 | **direct** | +| F-15 | **Mostly fixed in PR 5**: timeout, match caps, file-size cap. **A single pathological `re` call in the fallback is still unbounded** — Python offers no way to interrupt one | `apps/agent/src/dcode_agent/tools/grep.py` | medium | 5 | **direct** | | F-16 | ~~nginx serves the bundle uncompressed~~ **— fixed in PR 4**, measured on the built image at 760,088 → 269,497 bytes. Also fixed a duplicate `Cache-Control` header the first attempt introduced | `apps/frontend/nginx.conf.template` | low | 4 | **direct** | | F-17 | ~~No `.dockerignore` anywhere~~ **— fixed in PR 4**, one at the root and one for `apps/frontend` | repository root | low | 4 | **direct** | | F-21 | ~~A copy-pasteable setup block in § 10.1 contained placeholders, was pasted verbatim, and the placeholder became the credential — the API then refused to boot~~ **— fixed in PR 4** by rewriting the block to substitute its own values. The refusal was correct; the instructions were the defect | `Deploy.md` §10.1 | low | 4 | **direct** | @@ -759,21 +794,36 @@ Written from the code, not from a completed deployment — **nothing has been deployed yet** (§ 6 PR 4). Treat it as the intended sequence, and correct it against what actually happens. +**The configuration is committed.** `infra/railway/{api,agent-worker,frontend}.toml` +carry the build and deploy settings, and +[`infra/railway/README.md`](infra/railway/README.md) is the variable sheet — +which service needs what, and the two rules that fail silently. What follows is +the part that cannot live in a file. + **Services.** Six. Only `frontend` gets a public domain. | Service | Image | Notes | |---|---|---| -| `frontend` | `infra/docker/frontend.Dockerfile`, context `apps/frontend` | Public. Set `API_UPSTREAM=http://api.railway.internal:8000` and `DNS_RESOLVER` to Railway's internal resolver. `PORT` is injected. | +| `frontend` | `infra/docker/frontend.Dockerfile` | Public. Set `API_UPSTREAM` and `DNS_RESOLVER`; `PORT` is injected. Builds from the repository root — it used to build from `apps/frontend`, changed so no service needs a Root Directory. | | `api` | `infra/docker/api.Dockerfile`, context repo root | Private. Set `HOST=::` — Railway's private network is IPv6 and a service bound to `0.0.0.0` is reachable by nothing there. | | `agent-worker` | `infra/docker/railway-agent-worker.Dockerfile`, context repo root | Private. **Owns the only volume**, mounted at `WORKDIR_BASE`. Set `HOST=::`. | | Postgres | Railway **pgvector** template | Not the plain Postgres template — R-3. | | Redis | Railway template | | | RabbitMQ | Railway template | | -**Order.** Postgres/Redis/RabbitMQ first, then `api`, then run the migration -against it, then `agent-worker`, then `frontend`. The migration reads -`EMBEDDING_DIM` from the `api` service (§ 6 PR 1), so that variable has to be -right *before* it runs — afterwards the column dimension is fixed and changing +**Order.** Postgres/Redis/RabbitMQ first, then `api`, then `agent-worker`, then +`frontend`. + +The migration is **not** a manual step: `api.toml` runs it as a +`preDeployCommand`, in that container, before the deployment takes traffic. That +is where it belongs rather than where it is convenient — the migration reads +`EMBEDDING_DIM` to size the pgvector column, and that variable lives on the +`api` service (§ 6 PR 1). Running it anywhere else is how the column ends up at +a dimension nothing agrees with. `upgrade head` is idempotent, so every deploy +running it is fine. + +The consequence: **`EMBEDDING_DIM` must be correct before the first deploy of +`api`**, not before some later step. Afterwards the column is fixed and changing it means a new volume. **Variables that must agree across `api`, `agent-worker` and each other**: @@ -788,13 +838,22 @@ a `railway.json` or a bulk import is not. `AUTH_SESSION_SECRET`, `AUTH_COOKIE_SECURE=true`, `DOCS_ENABLED=false`. Railway serves HTTPS, so the Secure cookie is correct there. +**Health checks.** Point Railway's at `/readyz`, not `/healthz`. The shallow one +reports ok with every dependency down; the deep one covers the database, Redis +and the embedding-dimension agreement, and returns 503 with a per-check reason. +It skips the dimension check when `chunks` does not exist yet, so it is safe to +have configured before the migration step above. + **Then walk § 10's checklist.** The rows that need a browser need a browser. -**Two things this project has not solved and will meet here.** The first index -runs through the hosted embedding API for real — roughly 23 requests for -`psf/requests` — and that path has never been exercised (§ 6 PR 2). The second -is that cloned workdirs are never cleaned up (F-08), so the volume grows with -every distinct repository anyone indexes, forever. +**Three things to decide here rather than discover.** The first index runs +through the hosted embedding API for real — roughly 23 requests for +`psf/requests` — and that path has never been exercised (§ 6 PR 2). Set +`WORKDIR_MAX_REPOS` to something the volume can hold, since it defaults to +never evicting and read § 6 PR 5 for what eviction costs. And consider +`REPO_URL_ALLOWED_HOSTS`: on a gated demo the login wall is the real control, +but it is the only rule in the URL check that states what is *allowed*, and DNS +rebinding is not closed without it or network egress filtering. --- diff --git a/apps/agent/src/dcode_agent/graph.py b/apps/agent/src/dcode_agent/graph.py index 5dc76ee..b01a821 100644 --- a/apps/agent/src/dcode_agent/graph.py +++ b/apps/agent/src/dcode_agent/graph.py @@ -5,12 +5,15 @@ import logging import re from typing import Any, cast +from uuid import UUID +from dcode_shared.db.models import Repo from dcode_shared.observability import log_event from dcode_shared.reranker import RerankerClient, create_reranker_client from dcode_shared.schemas import CallDirection from dcode_shared.settings import shared_settings from langgraph.graph import END, START, StateGraph +from sqlalchemy import select from dcode_agent import groundedness from dcode_agent import state as state_module @@ -116,7 +119,15 @@ async def tool_call_node(state: AgentState) -> AgentState: state, tool_name, _jsonable_args(state.pending_tool_args), exc ) - cache_key = tool.cache_key(state.repo_id, args_model) + revision = await _index_revision(state) + # No revision, no cache. Sharing a key across corpus generations is the + # defect this closes; falling back to one when the revision cannot be read + # would reintroduce it in exactly the case nobody tests. + cache_key = ( + tool.cache_key(state.repo_id, args_model, index_revision=revision) + if revision is not None + else None + ) args_payload = args_model.model_dump(mode="json") await _emit_tool_call(state, tool_name, args_payload) @@ -127,7 +138,11 @@ async def tool_call_node(state: AgentState) -> AgentState: step=state.step_count + 1, tool=tool_name, ) - cached_payload = await _cache_get(state.runtime.get("tool_cache"), cache_key) + cached_payload = ( + await _cache_get(state.runtime.get("tool_cache"), cache_key) + if cache_key is not None + else None + ) cached = cached_payload is not None try: if cached: @@ -138,7 +153,7 @@ async def tool_call_node(state: AgentState) -> AgentState: except Exception as exc: # noqa: BLE001 — tool execution failed; degrade, not abort return await _record_tool_failure(state, tool_name, args_payload, exc) - if not cached: + if not cached and cache_key is not None: await _cache_set(state.runtime.get("tool_cache"), cache_key, json.dumps(result_payload)) log_event( logger, @@ -1413,6 +1428,34 @@ async def _emit_partial_answer(state: AgentState, delta: str) -> None: await emitter.emit_partial_answer(delta) +async def _index_revision(state: AgentState) -> int | None: + """The repository's corpus generation, read once per query and memoised. + + Once per query rather than per tool call, so every tool in one walk sees a + consistent corpus even if a re-index lands mid-answer. + + Returns None when it cannot be read — no database session, no such + repository, or a query error. The caller bypasses the cache in that case. + """ + if state.index_revision is not None: + return state.index_revision + + db = state.runtime.get("db") + if db is None: + return None + try: + result = await db.execute( + select(Repo.index_revision).where(Repo.id == UUID(state.repo_id)) + ) + revision = result.scalar_one_or_none() + except Exception: # noqa: BLE001 — an unreadable revision degrades to no cache + return None + if revision is None: + return None + state.index_revision = int(revision) + return state.index_revision + + async def _cache_get(cache: Any, key: str) -> str | None: if cache is None: return None diff --git a/apps/agent/src/dcode_agent/main.py b/apps/agent/src/dcode_agent/main.py index 6ca0b32..cfea312 100644 --- a/apps/agent/src/dcode_agent/main.py +++ b/apps/agent/src/dcode_agent/main.py @@ -8,7 +8,7 @@ from dcode_shared.db.session import SessionLocal from dcode_shared.events import CitationEvent -from dcode_shared.internal import INTERNAL_API_KEY_HEADER +from dcode_shared.internal import INTERNAL_API_KEY_HEADER, internal_api_key_error from dcode_shared.schemas import QueryRequest from fastapi import FastAPI, Header, HTTPException, status from fastapi.responses import StreamingResponse @@ -24,6 +24,13 @@ @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: + # The agent validates the same shared key that protects it. Running on the + # published placeholder means its /internal/query surface is open to + # anything that can reach it on the private network. + key_error = internal_api_key_error(agent_settings.internal_api_key, strict=False) + if key_error is not None: + raise RuntimeError(key_error) + app.state.tool_registry = default_registry() app.state.compiled_graph = graph.build_graph() app.state.tool_cache = Redis.from_url(agent_settings.redis_url, decode_responses=True) diff --git a/apps/agent/src/dcode_agent/state.py b/apps/agent/src/dcode_agent/state.py index c99ddaa..7bf7317 100644 --- a/apps/agent/src/dcode_agent/state.py +++ b/apps/agent/src/dcode_agent/state.py @@ -93,4 +93,9 @@ class AgentState: groundedness_score: float | None = None final_answer: str | None = None error: str | None = None + # The repository's corpus generation, read once per query and folded into + # every tool cache key. `None` means it could not be read, and the tool + # cache is bypassed for the request rather than shared with another + # generation — see `_index_revision` in graph.py. + index_revision: int | None = None runtime: dict[str, Any] = field(default_factory=dict, repr=False) diff --git a/apps/agent/src/dcode_agent/tools/base.py b/apps/agent/src/dcode_agent/tools/base.py index 703d0e7..f2bc0a6 100644 --- a/apps/agent/src/dcode_agent/tools/base.py +++ b/apps/agent/src/dcode_agent/tools/base.py @@ -33,9 +33,14 @@ class Tool(ABC, Generic[ArgsT, ResultT]): async def execute(self, repo_id: str, args: ArgsT) -> ResultT: """Execute the tool against the live index for `repo_id`.""" - def cache_key(self, repo_id: str, args: ArgsT) -> str: + def cache_key(self, repo_id: str, args: ArgsT, *, index_revision: int) -> str: """Redis cache key for this invocation in the 24-hour tool namespace.""" - return tool_cache_key(self.name, repo_id, args.model_dump(mode="json")) + return tool_cache_key( + self.name, + repo_id, + args.model_dump(mode="json"), + index_revision=index_revision, + ) class ToolRegistry: diff --git a/apps/agent/src/dcode_agent/tools/common.py b/apps/agent/src/dcode_agent/tools/common.py index 657dce7..5c18a29 100644 --- a/apps/agent/src/dcode_agent/tools/common.py +++ b/apps/agent/src/dcode_agent/tools/common.py @@ -36,10 +36,23 @@ async def fetch_internal_json( def repo_root(repo_id: str) -> Path: - """Return the cloned repo root for one indexed repository.""" + """Return the cloned repo root for one indexed repository. + + The message matters. A repository can be `ready` in Postgres — chunks, + symbols and edges all present, so search and graph tools work — while its + cloned tree is gone, because workdir retention and index state are separate + concerns (see the worker's prune step). In that situation `read_file`, + `grep` and `list_directory` are the only things that fail, and a bare + FileNotFoundError sends the reader looking for a missing file rather than a + missing checkout. + """ root = (Path(agent_settings.workdir_base).expanduser() / repo_id).resolve() if not root.exists() or not root.is_dir(): - raise FileNotFoundError(f"indexed repo workdir not found for repo_id={repo_id}") + raise FileNotFoundError( + f"repository source is not on disk for repo_id={repo_id}. " + "Search and call-graph tools still work; file-reading tools need the " + "checkout, which is restored by re-indexing this repository." + ) return root diff --git a/apps/agent/src/dcode_agent/tools/grep.py b/apps/agent/src/dcode_agent/tools/grep.py index e007d38..a70d6c3 100644 --- a/apps/agent/src/dcode_agent/tools/grep.py +++ b/apps/agent/src/dcode_agent/tools/grep.py @@ -8,6 +8,7 @@ import json import re import shutil +import time from pathlib import Path from dcode_shared.schemas import Location @@ -27,6 +28,19 @@ class GrepResult(BaseModel): locations: list[Location] +# Bounds on a tool whose argument is a user-influenced pattern. +# +# `.` over a large repository produced hundreds of MB of JSON, buffered whole in +# the agent process and then serialised into Redis and the SSE state. None of +# that is useful to an answer: a pattern matching everything has told you +# nothing, and the first few hundred hits are as informative as the first few +# hundred thousand. +_MATCH_LIMIT = 500 +_PER_FILE_MATCH_LIMIT = 20 +_MAX_FILE_BYTES = 1_000_000 +_GREP_TIMEOUT_SECONDS = 20.0 + + class GrepTool(Tool[GrepArgs, GrepResult]): name = "grep" description = "Exact pattern search via ripgrep — cheap and precise." @@ -46,13 +60,26 @@ async def _grep_with_ripgrep(root: Path, pattern: str) -> list[Location]: "rg", "--json", "-n", + # Bounded at the source rather than after the fact: rg stops looking + # instead of the agent reading everything and throwing most away. + f"--max-count={_PER_FILE_MATCH_LIMIT}", + f"--max-filesize={_MAX_FILE_BYTES}", "-e", pattern, str(root), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - stdout, stderr = await process.communicate() + try: + stdout, stderr = await asyncio.wait_for( + process.communicate(), timeout=_GREP_TIMEOUT_SECONDS + ) + except TimeoutError as exc: + # The clone path has a timeout and this did not, so a pathological + # pattern had no ceiling at all. + process.kill() + await process.wait() + raise RuntimeError(f"grep timed out after {_GREP_TIMEOUT_SECONDS:.0f}s") from exc if process.returncode not in (0, 1): error = stderr.decode("utf-8", errors="replace").strip() raise RuntimeError(f"rg failed: {error or process.returncode}") @@ -73,22 +100,41 @@ async def _grep_with_ripgrep(root: Path, pattern: str) -> list[Location]: chunk_id=None, ) ) + if len(matches) >= _MATCH_LIMIT: + break return matches def _grep_with_python(root: Path, pattern: str) -> list[Location]: + """Fallback for when ripgrep is absent. Bounded, but not fully. + + Honest limit: Python has no way to time out a single `re` call, so a pattern + with catastrophic backtracking against one long line can still block the + event loop. What is bounded is everything countable — total matches, matches + per file, file size, and a wall-clock deadline checked between files. + + The agent image installs ripgrep, so this path runs only where it is + missing: a host-run process or a stripped image. That is why the residual + risk is recorded rather than solved with a subprocess. + """ regex = re.compile(pattern) + deadline = time.monotonic() + _GREP_TIMEOUT_SECONDS matches: list[Location] = [] for path in sorted(root.rglob("*")): + if len(matches) >= _MATCH_LIMIT or time.monotonic() > deadline: + break if not path.is_file(): continue if any(part in _SKIP_DIRS for part in path.relative_to(root).parts[:-1]): continue try: + if path.stat().st_size > _MAX_FILE_BYTES: + continue lines = path.read_text(encoding="utf-8").splitlines() except (OSError, UnicodeDecodeError): continue relative = path.relative_to(root).as_posix() + in_file = 0 for line_number, line in enumerate(lines, start=1): if regex.search(line): matches.append( @@ -99,4 +145,7 @@ def _grep_with_python(root: Path, pattern: str) -> list[Location]: chunk_id=None, ) ) + in_file += 1 + if in_file >= _PER_FILE_MATCH_LIMIT or len(matches) >= _MATCH_LIMIT: + break return matches diff --git a/apps/agent/tests/test_graph.py b/apps/agent/tests/test_graph.py index f097c37..54e5f21 100644 --- a/apps/agent/tests/test_graph.py +++ b/apps/agent/tests/test_graph.py @@ -424,6 +424,30 @@ async def test_multihop_expands_three_distinct_hybrid_seeds() -> None: ) +class FakeRevisionDb: + """Answers the one SELECT the tool cache key needs. + + Present in these tests because the tool cache is now keyed by the + repository's corpus generation, and a state with no database session + deliberately bypasses the cache rather than sharing a key across + generations. + """ + + def __init__(self, revision: int) -> None: + self.revision = revision + self.queries = 0 + + async def execute(self, _statement: object) -> object: + self.queries += 1 + revision = self.revision + + class _Result: + def scalar_one_or_none(self) -> int: + return revision + + return _Result() + + async def test_tool_call_node_executes_and_then_hits_cache(caplog) -> None: caplog.set_level(logging.INFO, logger="dcode.agent.graph") tool = DummyTool() @@ -431,12 +455,19 @@ async def test_tool_call_node_executes_and_then_hits_cache(caplog) -> None: emitter = FakeEmitter() cache: dict[str, str] = {} repo_id = str(uuid4()) + db = FakeRevisionDb(revision=3) + runtime = { + "tool_registry": registry, + "tool_cache": cache, + "emitter": emitter, + "db": db, + } state = AgentState( repo_id=repo_id, query="Where is `HTTPBasicAuth` defined?", pending_tool_name="find_definition", pending_tool_args={"symbol": "HTTPBasicAuth"}, - runtime={"tool_registry": registry, "tool_cache": cache, "emitter": emitter}, + runtime=runtime, ) first = await tool_call_node(state) @@ -446,7 +477,7 @@ async def test_tool_call_node_executes_and_then_hits_cache(caplog) -> None: query=state.query, pending_tool_name="find_definition", pending_tool_args={"symbol": "HTTPBasicAuth"}, - runtime={"tool_registry": registry, "tool_cache": cache, "emitter": emitter}, + runtime=runtime, ) ) @@ -1302,3 +1333,97 @@ async def rerank(self, query: str, passages: list[str]) -> list[float]: catalog = await graph_module._ranked_evidence_catalog(state) assert catalog["C1"].display_token == "src/requests/mod0.py:1" + + +async def test_reindexing_invalidates_the_tool_cache() -> None: + """The defect F-07 records: a re-index left cached results addressable. + + The key carried (tool, repo_id, args) and nothing about which generation of + the corpus produced the answer, so for 24 hours after re-indexing the agent + answered from the previous one. Nothing failed and nothing logged. + """ + tool = DummyTool() + registry = _registry(tool) + cache: dict[str, str] = {} + repo_id = str(uuid4()) + + def state_for(revision: int) -> AgentState: + return AgentState( + repo_id=repo_id, + query="Where is `HTTPBasicAuth` defined?", + pending_tool_name="find_definition", + pending_tool_args={"symbol": "HTTPBasicAuth"}, + runtime={ + "tool_registry": registry, + "tool_cache": cache, + "emitter": FakeEmitter(), + "db": FakeRevisionDb(revision=revision), + }, + ) + + first = await tool_call_node(state_for(1)) + again = await tool_call_node(state_for(1)) + after_reindex = await tool_call_node(state_for(2)) + + assert first.observations[0]["cached"] is False + assert again.observations[0]["cached"] is True, "same generation should still hit" + assert after_reindex.observations[0]["cached"] is False, "a new generation must miss" + assert tool.calls == 2 + assert len(cache) == 2 + + +async def test_an_unreadable_revision_bypasses_the_cache_rather_than_sharing_one() -> None: + """No revision, no cache. + + Falling back to a revision-less key would reintroduce the cross-generation + collision in exactly the situation nobody exercises, so the request pays for + a real tool call instead. + """ + tool = DummyTool() + registry = _registry(tool) + cache: dict[str, str] = {} + repo_id = str(uuid4()) + + def state_without_db() -> AgentState: + return AgentState( + repo_id=repo_id, + query="q", + pending_tool_name="find_definition", + pending_tool_args={"symbol": "HTTPBasicAuth"}, + runtime={"tool_registry": registry, "tool_cache": cache, "emitter": FakeEmitter()}, + ) + + first = await tool_call_node(state_without_db()) + second = await tool_call_node(state_without_db()) + + assert tool.calls == 2 + assert cache == {} + assert first.tool_calls[0]["cache_key"] is None + assert second.observations[0]["cached"] is False + + +async def test_the_revision_is_read_once_per_query() -> None: + """Read per query, not per tool call, so one walk sees one corpus.""" + tool = DummyTool() + registry = _registry(tool) + db = FakeRevisionDb(revision=7) + state = AgentState( + repo_id=str(uuid4()), + query="q", + pending_tool_name="find_definition", + pending_tool_args={"symbol": "HTTPBasicAuth"}, + runtime={ + "tool_registry": registry, + "tool_cache": {}, + "emitter": FakeEmitter(), + "db": db, + }, + ) + + after = await tool_call_node(state) + after.pending_tool_name = "find_definition" + after.pending_tool_args = {"symbol": "Session"} + await tool_call_node(after) + + assert db.queries == 1 + assert after.index_revision == 7 diff --git a/apps/agent/tests/test_grep_bounds.py b/apps/agent/tests/test_grep_bounds.py new file mode 100644 index 0000000..abc5ab1 --- /dev/null +++ b/apps/agent/tests/test_grep_bounds.py @@ -0,0 +1,102 @@ +"""Bounds on the grep tool (F-15). + +The tool's argument is a pattern the planner derives from a user question, and +it had no ceiling of any kind: no timeout, no match cap, and the whole of +ripgrep's stdout buffered in the agent process before anything looked at it. +""" + +import re +from pathlib import Path + +from dcode_agent.tools import grep as grep_module + + +def _repo(tmp_path: Path, files: dict[str, str]) -> Path: + for name, content in files.items(): + target = tmp_path / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return tmp_path + + +def test_python_fallback_caps_total_matches(tmp_path: Path) -> None: + """A pattern matching everything has told you nothing. + + The first few hundred hits are as informative as the first few hundred + thousand, and the difference is hundreds of MB through Redis and the SSE + state. + """ + files = {f"mod{index}.py": "x\n" * 200 for index in range(20)} + root = _repo(tmp_path, files) + + matches = grep_module._grep_with_python(root, "x") + + assert len(matches) <= grep_module._MATCH_LIMIT + + +def test_python_fallback_caps_matches_per_file(tmp_path: Path) -> None: + root = _repo(tmp_path, {"one.py": "hit\n" * 500}) + + matches = grep_module._grep_with_python(root, "hit") + + assert len(matches) == grep_module._PER_FILE_MATCH_LIMIT + + +def test_python_fallback_skips_a_file_over_the_size_cap(tmp_path: Path) -> None: + root = _repo( + tmp_path, + { + "small.py": "needle\n", + "huge.py": "needle\n" + ("padding\n" * (grep_module._MAX_FILE_BYTES // 8 + 10)), + }, + ) + + matches = grep_module._grep_with_python(root, "needle") + + assert [match.file_path for match in matches] == ["small.py"] + + +def test_python_fallback_stops_at_the_deadline( + tmp_path: Path, monkeypatch: object +) -> None: + """The wall-clock bound is checked between files. + + A single `re` call cannot be interrupted in Python, so this is what is + actually enforceable — and the tool's docstring says so rather than + implying the fallback is fully bounded. + """ + root = _repo(tmp_path, {f"m{index}.py": "needle\n" for index in range(50)}) + + ticks = iter([0.0] + [999.0] * 200) + monkeypatch.setattr(grep_module.time, "monotonic", lambda: next(ticks)) # type: ignore[attr-defined] + + matches = grep_module._grep_with_python(root, "needle") + + assert matches == [], "an expired deadline must stop the walk" + + +def test_the_ripgrep_invocation_carries_its_own_bounds() -> None: + """Bounded at the source, not after the fact. + + Asserted on the argument list because the alternative is reading everything + ripgrep produces and discarding most of it, which is the behaviour being + replaced. + """ + source = Path(grep_module.__file__).read_text(encoding="utf-8") + assert "--max-count=" in source + assert "--max-filesize=" in source + assert "asyncio.wait_for" in source + + +def test_a_pattern_that_is_not_a_regex_surfaces_as_a_tool_error(tmp_path: Path) -> None: + """The planner can produce an invalid pattern; that is a tool failure. + + The agent records tool failures and degrades to synthesis, so this must + raise rather than return an empty result that reads like "no matches". + """ + root = _repo(tmp_path, {"a.py": "x\n"}) + try: + grep_module._grep_with_python(root, "(unclosed") + except re.error: + return + raise AssertionError("an invalid pattern should not look like zero matches") diff --git a/apps/agent/tests/test_tools_registry.py b/apps/agent/tests/test_tools_registry.py index cfce860..e129d3d 100644 --- a/apps/agent/tests/test_tools_registry.py +++ b/apps/agent/tests/test_tools_registry.py @@ -42,8 +42,8 @@ def test_cache_key_is_deterministic_and_namespaced() -> None: tool = default_registry().get("search_code") assert tool is not None args = tool.ArgsSchema(query="x", k=5) - key_a = tool.cache_key("repo-1", args) - key_b = tool.cache_key("repo-1", args) + key_a = tool.cache_key("repo-1", args, index_revision=0) + key_b = tool.cache_key("repo-1", args, index_revision=0) assert key_a == key_b assert key_a.startswith("tool:search_code:repo-1:") diff --git a/apps/api/src/dcode_api/main.py b/apps/api/src/dcode_api/main.py index c84966c..0e0c6d6 100644 --- a/apps/api/src/dcode_api/main.py +++ b/apps/api/src/dcode_api/main.py @@ -8,10 +8,11 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager -from fastapi import Depends, FastAPI +from dcode_shared.internal import internal_api_key_error +from fastapi import Depends, FastAPI, Response, status from fastapi.middleware.cors import CORSMiddleware -from dcode_api import deps +from dcode_api import deps, readiness from dcode_api.auth import auth_configuration_error, require_session from dcode_api.routes import auth, inspector, internal, query, repos from dcode_api.settings import api_settings @@ -22,7 +23,12 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: # Refuse to start a gate that cannot be enforced. Serving the workbench # openly because a secret was missing is the failure this prevents, and it # is one that looks fine from the outside. - misconfigured = auth_configuration_error() + misconfigured = auth_configuration_error() or internal_api_key_error( + api_settings.internal_api_key, + # The gate being on is this process's best available signal that it is + # a real deployment rather than someone's laptop. + strict=api_settings.auth_enabled, + ) if misconfigured is not None: raise RuntimeError(misconfigured) @@ -78,9 +84,33 @@ def create_app() -> FastAPI: @application.get("/healthz", tags=["meta"]) async def healthz() -> dict[str, str]: - """Shallow liveness probe — does not check dependent services.""" + """Shallow liveness probe — does not check dependent services. + + Deliberately shallow: a liveness probe that fails when a dependency is + down gets the process restarted, which fixes nothing and loses the logs. + Use /readyz to decide whether it can serve. + """ return {"status": "ok"} + @application.get("/readyz", tags=["meta"]) + async def readyz(response: Response) -> dict[str, object]: + """Deep probe: database, Redis, and the embedding-dimension agreement. + + 503 when anything fails, with a per-check reason. Open like /healthz — + it reports whether this process works, not anything about the corpus. + """ + checks = await readiness.run_checks() + ready = all(check.ok for check in checks) + if not ready: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + return { + "status": "ready" if ready else "not ready", + "checks": [ + {"name": check.name, "ok": check.ok, "detail": check.detail} + for check in checks + ], + } + return application diff --git a/apps/api/src/dcode_api/readiness.py b/apps/api/src/dcode_api/readiness.py new file mode 100644 index 0000000..5818b04 --- /dev/null +++ b/apps/api/src/dcode_api/readiness.py @@ -0,0 +1,109 @@ +"""Readiness checks for the gateway (F-10). + +`/healthz` answers `ok` unconditionally — its own docstring says it does not +check dependent services. That is correct for a liveness probe and useless for +deciding whether this process can serve a request: it reported healthy with +Postgres, Redis and RabbitMQ all down, and the first user request was what +discovered it. + +The dimension check is here rather than in a runbook because of Deploy.md §5.4: +`chunks.embedding` is fixed at migration time, and a service configured for a +different dimension produces no error — the worker's inserts fail, or dense +search runs a stub all-zero query vector against real stored vectors and +silently returns noise. A probe is the only place that catches it before a user +does. +""" + +from dataclasses import dataclass + +from dcode_shared.db.session import SessionLocal +from redis.exceptions import RedisError +from sqlalchemy import text + +from dcode_api.deps import get_redis +from dcode_api.settings import api_settings + + +@dataclass(frozen=True) +class Check: + """One readiness check. `detail` is why it failed, never a secret.""" + + name: str + ok: bool + detail: str | None = None + + +async def run_checks() -> list[Check]: + """Every check, always all of them. + + Not short-circuited on the first failure: an operator looking at this wants + to know whether one thing is broken or everything is, and returning after + the first one turns "the database is down" into a report that says nothing + about Redis. + """ + return [await _check_database(), await _check_redis(), await _check_embedding_dim()] + + +async def _check_database() -> Check: + try: + async with SessionLocal() as session: + await session.execute(text("SELECT 1")) + except Exception as exc: # noqa: BLE001 — any failure is not-ready + return Check("database", False, _reason(exc)) + return Check("database", True) + + +async def _check_redis() -> Check: + try: + redis = await get_redis() + await redis.ping() + except (RedisError, OSError) as exc: + return Check("redis", False, _reason(exc)) + except Exception as exc: # noqa: BLE001 + return Check("redis", False, _reason(exc)) + return Check("redis", True) + + +async def _check_embedding_dim() -> Check: + """Compare the configured dimension against the live column. + + Skipped rather than failed when the table is empty or absent: before the + first migration there is nothing to disagree with, and reporting not-ready + would block a deploy that has not reached the migration step yet. + """ + configured = api_settings.embedding_dim + try: + async with SessionLocal() as session: + result = await session.execute( + text( + "SELECT atttypmod FROM pg_attribute " + "WHERE attrelid = 'chunks'::regclass AND attname = 'embedding'" + ) + ) + live = result.scalar_one_or_none() + except Exception: # noqa: BLE001 — no table yet is not a mismatch + return Check("embedding_dim", True, f"not checked (configured {configured})") + + if live is None: + return Check("embedding_dim", True, f"not checked (configured {configured})") + if int(live) != configured: + return Check( + "embedding_dim", + False, + ( + f"EMBEDDING_DIM is {configured} but chunks.embedding is vector({int(live)}). " + "The column is fixed at migration time; this cannot be reconciled by " + "restarting." + ), + ) + return Check("embedding_dim", True) + + +def _reason(exc: Exception) -> str: + """Exception type and message, trimmed. + + A connection error can carry a DSN, so the detail is bounded rather than + passed through — a readiness probe is often reachable by more people than + the logs are. + """ + return f"{type(exc).__name__}: {exc}"[:200] diff --git a/apps/api/src/dcode_api/routes/repos.py b/apps/api/src/dcode_api/routes/repos.py index e9f3f50..acab593 100644 --- a/apps/api/src/dcode_api/routes/repos.py +++ b/apps/api/src/dcode_api/routes/repos.py @@ -1,8 +1,10 @@ """Public repository submission and indexing-status endpoints.""" +import asyncio import ipaddress import json import re +import socket from collections.abc import Awaitable, Callable from urllib.parse import urlparse from uuid import UUID @@ -26,11 +28,13 @@ from sqlalchemy.ext.asyncio import AsyncSession from dcode_api.deps import get_db, get_index_job_publisher, get_redis +from dcode_api.settings import api_settings router = APIRouter(tags=["repos"]) _SCP_LIKE_GIT_URL = re.compile(r"^[\w.-]+@[\w.-]+:[\w./-]+(?:\.git)?$") _ALLOWED_URL_SCHEMES = {"https", "http", "ssh", "git"} +_DNS_TIMEOUT_SECONDS = 3.0 @router.post( @@ -53,13 +57,11 @@ async def submit_repo( is durable, the repo is marked failed rather than left queued forever. """ repo_url = body.url.strip() - if not _is_supported_git_url(repo_url): + rejection = await _reject_repo_url(repo_url) + if rejection is not None: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail={ - "code": "INVALID_REPO_URL", - "message": "Expected an http(s), ssh, git, or git@host:path Git URL.", - }, + detail={"code": "INVALID_REPO_URL", "message": rejection}, ) existing = await _find_reusable_repo(db, repo_url) @@ -215,20 +217,105 @@ async def repo_status( ) -def _is_supported_git_url(url: str) -> bool: +async def _reject_repo_url(url: str) -> str | None: + """Why this URL may not be cloned, or None. The message reaches the caller. + + Three checks beyond the syntax one, in order of how much they buy: + + 1. **No credentials in the URL.** `https://user:token@host/...` was accepted + and then persisted into `repos.url` and echoed back by the status route, + so a token handed to this endpoint became readable by anyone who could + read a repository's status. Rejected rather than stripped: quietly + mutating what someone submitted is worse than saying no. + 2. **An optional host allowlist.** Empty by default, which keeps any public + host acceptable. Set `REPO_URL_ALLOWED_HOSTS` and it becomes the strong + control — everything else here is a filter on what is obviously wrong, + and this is the only rule that states what is right. + 3. **Resolution.** The literal-IP check never fired for a *name*, so + `evil.example.com` pointing at `169.254.169.254` or `10.0.0.1` passed. + Every resolved address is now checked. + + What check 3 does **not** close is rebinding: git resolves the name again + when it clones, and a DNS answer that changes in between defeats this. The + fix for that is egress filtering at the network, not more code here, and it + is recorded as such in Deploy.md rather than implied to be handled. + """ if not url: - return False + return "Expected a Git URL." + scp_like = _SCP_LIKE_GIT_URL.match(url) if scp_like: host = url.split("@", 1)[1].split(":", 1)[0] - return _is_allowed_remote_host(host) + else: + parsed = urlparse(url) + if parsed.scheme not in _ALLOWED_URL_SCHEMES: + return "Expected an http(s), ssh, git, or git@host:path Git URL." + if parsed.password is not None or (parsed.username and parsed.scheme != "ssh"): + return ( + "Remove the credentials from the URL. This endpoint indexes public " + "repositories, and a submitted URL is stored and shown in status responses." + ) + if not parsed.hostname: + return "Expected an http(s), ssh, git, or git@host:path Git URL." + host = parsed.hostname + + if not _is_allowed_remote_host(host): + return "That host is not a permitted clone target." + + allowlist = api_settings.repo_url_allowed_hosts_list + normalized = host.strip().strip("[]").rstrip(".").lower() + if allowlist and not any( + normalized == allowed or normalized.endswith(f".{allowed}") for allowed in allowlist + ): + return f"Only these hosts may be cloned: {', '.join(sorted(allowlist))}." - parsed = urlparse(url) - if parsed.scheme not in _ALLOWED_URL_SCHEMES: - return False - if not parsed.hostname: - return False - return _is_allowed_remote_host(parsed.hostname) + unreachable = await _reject_resolved_addresses(normalized) + if unreachable is not None: + return unreachable + return None + + +async def resolve_host(host: str) -> list[str]: + """Addresses `host` resolves to. + + A module-level function rather than an inline `loop.getaddrinfo` so it can + be replaced in tests. Without a seam here, every test that submits a + repository URL performs a real DNS lookup — which makes the suite depend on + the network, and makes CI slower and occasionally wrong for a reason that + has nothing to do with the code under test. + """ + loop = asyncio.get_running_loop() + infos = await asyncio.wait_for( + loop.getaddrinfo(host, None, proto=socket.IPPROTO_TCP), + timeout=_DNS_TIMEOUT_SECONDS, + ) + return [str(info[4][0]) for info in infos] + + +async def _reject_resolved_addresses(host: str) -> str | None: + """Resolve `host` and refuse if any answer is a non-public address.""" + try: + ipaddress.ip_address(host) + except ValueError: + pass + else: + # A literal was already checked by _is_allowed_remote_host. + return None + + try: + addresses = await resolve_host(host) + except (TimeoutError, OSError): + # Refusing here would make an unrelated DNS hiccup look like a rejected + # URL, and the worker's clone would fail anyway with a clearer reason. + return None + + for address in addresses: + if not _is_allowed_remote_host(address): + return ( + f"{host} resolves to a private or reserved address " + "and will not be cloned." + ) + return None def _is_allowed_remote_host(host: str) -> bool: diff --git a/apps/api/src/dcode_api/settings.py b/apps/api/src/dcode_api/settings.py index 67683f9..2b3521f 100644 --- a/apps/api/src/dcode_api/settings.py +++ b/apps/api/src/dcode_api/settings.py @@ -38,6 +38,17 @@ class APISettings(SharedSettings): # Useful locally, so on by default; prod compose turns it off. docs_enabled: bool = True + # Comma-separated clone-target allowlist. Empty means any public host, which + # is what a demo indexing arbitrary open-source repositories needs. Set it + # and it becomes the only rule in the URL check that states what is + # *allowed* rather than what is obviously wrong. Subdomains match. + repo_url_allowed_hosts: str = "" + + @property + def repo_url_allowed_hosts_list(self) -> list[str]: + """Comma-separated REPO_URL_ALLOWED_HOSTS env var → lowercased list.""" + return [h.strip().lower() for h in self.repo_url_allowed_hosts.split(",") if h.strip()] + @property def cors_origins_list(self) -> list[str]: """Comma-separated CORS_ORIGINS env var → list.""" diff --git a/apps/api/tests/conftest.py b/apps/api/tests/conftest.py new file mode 100644 index 0000000..f118206 --- /dev/null +++ b/apps/api/tests/conftest.py @@ -0,0 +1,24 @@ +"""Keep the API test suite off the network. + +`POST /repos` resolves the submitted hostname to check that it is not a private +address (F-06). Left alone, every test that submits a URL would perform a real +DNS lookup: the suite would need the network, run slower, and fail occasionally +for a reason unrelated to what it is testing. + +This stubs the lookup to one public address by default. Tests that care about +resolution replace it themselves. +""" + +from collections.abc import Iterator + +import pytest +from dcode_api.routes import repos + + +@pytest.fixture(autouse=True) +def stub_host_resolution(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + async def public_address(_host: str) -> list[str]: + return ["140.82.121.4"] + + monkeypatch.setattr(repos, "resolve_host", public_address) + yield diff --git a/apps/api/tests/test_readiness.py b/apps/api/tests/test_readiness.py new file mode 100644 index 0000000..c383919 --- /dev/null +++ b/apps/api/tests/test_readiness.py @@ -0,0 +1,139 @@ +"""/readyz (F-10). + +/healthz answers `ok` with every dependency down — correct for liveness, +useless for deciding whether this process can serve. These cover the deep probe +and, in particular, the embedding-dimension check that Deploy.md §5.4 asks for, +because a dimension mismatch is the one misconfiguration in this stack that +produces no error at all. +""" + +import pytest +from dcode_api import readiness +from dcode_api.main import app +from fastapi.testclient import TestClient + + +def _readyz(monkeypatch: pytest.MonkeyPatch, checks: list[readiness.Check]) -> tuple[int, dict]: + async def stub() -> list[readiness.Check]: + return checks + + monkeypatch.setattr(readiness, "run_checks", stub) + response = TestClient(app).get("/readyz") + return response.status_code, response.json() + + +def test_ready_when_every_check_passes(monkeypatch: pytest.MonkeyPatch) -> None: + code, body = _readyz( + monkeypatch, + [readiness.Check("database", True), readiness.Check("redis", True)], + ) + assert code == 200 + assert body["status"] == "ready" + + +def test_not_ready_is_a_503_with_a_reason(monkeypatch: pytest.MonkeyPatch) -> None: + """503, not 200-with-a-body. + + A load balancer reads the status line. Reporting a failure only in the JSON + would keep sending traffic to a process that cannot serve it. + """ + code, body = _readyz( + monkeypatch, + [ + readiness.Check("database", False, "OperationalError: connection refused"), + readiness.Check("redis", True), + ], + ) + assert code == 503 + assert body["status"] == "not ready" + failed = [check for check in body["checks"] if not check["ok"]] + assert len(failed) == 1 + assert "connection refused" in failed[0]["detail"] + + +def test_healthz_stays_shallow_and_open() -> None: + """Unchanged on purpose. + + A liveness probe that fails when a dependency is down gets the process + restarted, which fixes nothing and loses the logs. + """ + response = TestClient(app).get("/healthz") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +async def test_a_dimension_mismatch_is_not_ready(monkeypatch: pytest.MonkeyPatch) -> None: + """The failure mode §5.4 describes, which nothing else catches. + + `chunks.embedding` is fixed at migration time. A service configured for + another dimension either fails every insert or searches real vectors with a + stub all-zero query vector and returns noise — with no error either way. + """ + from dcode_api.settings import api_settings + + monkeypatch.setattr(api_settings, "embedding_dim", 768) + monkeypatch.setattr(readiness, "SessionLocal", _session_returning(1024)) + + check = await readiness._check_embedding_dim() + + assert check.ok is False + assert check.detail is not None + assert "768" in check.detail and "1024" in check.detail + assert "restarting" in check.detail, "the reason should say a restart will not fix it" + + +async def test_a_matching_dimension_is_ready(monkeypatch: pytest.MonkeyPatch) -> None: + from dcode_api.settings import api_settings + + monkeypatch.setattr(api_settings, "embedding_dim", 768) + monkeypatch.setattr(readiness, "SessionLocal", _session_returning(768)) + + assert (await readiness._check_embedding_dim()).ok is True + + +async def test_a_missing_chunks_table_is_skipped_not_failed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Before the first migration there is nothing to disagree with. + + Reporting not-ready would block a deployment that has not reached the + migration step yet, which is a real ordering in the Railway runbook. + """ + monkeypatch.setattr(readiness, "SessionLocal", _session_raising()) + + check = await readiness._check_embedding_dim() + + assert check.ok is True + assert check.detail is not None and "not checked" in check.detail + + +def _session_returning(dimension: int | None) -> object: + class _Result: + def scalar_one_or_none(self) -> int | None: + return dimension + + class _Session: + async def __aenter__(self) -> "_Session": + return self + + async def __aexit__(self, *_: object) -> None: + return None + + async def execute(self, _statement: object) -> _Result: + return _Result() + + return _Session + + +def _session_raising() -> object: + class _Session: + async def __aenter__(self) -> "_Session": + return self + + async def __aexit__(self, *_: object) -> None: + return None + + async def execute(self, _statement: object) -> object: + raise RuntimeError('relation "chunks" does not exist') + + return _Session diff --git a/apps/api/tests/test_repo_url_guard.py b/apps/api/tests/test_repo_url_guard.py new file mode 100644 index 0000000..e54ac23 --- /dev/null +++ b/apps/api/tests/test_repo_url_guard.py @@ -0,0 +1,193 @@ +"""Clone-target validation (F-06). + +The literal-IP filter these tests sit on top of already existed. What did not +was any check on a *name*, so the interesting cases here are the ones that +would previously have been accepted. +""" + +import uuid + +import pytest +from dcode_api.deps import get_db, get_index_job_publisher +from dcode_api.main import app +from dcode_api.routes import repos +from fastapi.testclient import TestClient + + +class _EmptyIndex: + """A session with no existing repositories, permissive on writes. + + Permissive on purpose: a *rejected* URL must not reach the write path, and a + test that asserts acceptance needs the write path to succeed. Asserting + inside the fake conflates the two — it turns "accepted" into a crash, which + is indistinguishable from a different bug. + """ + + def __init__(self) -> None: + self.added: list[object] = [] + + async def execute(self, _statement: object) -> object: + class _Result: + def scalars(self) -> "_Result": + return self + + def all(self) -> list[object]: + return [] + + return _Result() + + def add(self, row: object) -> None: + self.added.append(row) + if getattr(row, "id", None) is None: + row.id = uuid.uuid4() + + async def flush(self) -> None: + return None + + async def commit(self) -> None: + return None + + +def _client(session: _EmptyIndex, published: list[str]) -> TestClient: + async def publish(_repo_id: object, url: str) -> None: + published.append(url) + + app.dependency_overrides[get_db] = lambda: session + app.dependency_overrides[get_index_job_publisher] = lambda: publish + return TestClient(app) + + +def _submit(url: str) -> tuple[int, str]: + """Submit one URL. Returns (status, rejection message). + + Also asserts the invariant that matters more than either: a rejected URL + creates no row and queues no job. + """ + session, published = _EmptyIndex(), [] + client = _client(session, published) + try: + response = client.post("/api/v1/repos", json={"url": url}) + finally: + app.dependency_overrides.clear() + + if response.status_code == 400: + assert session.added == [], "a rejected URL must not create a repo row" + assert published == [], "a rejected URL must not be queued for cloning" + + detail = response.json().get("detail", {}) + return response.status_code, str(detail.get("message", "")) if isinstance(detail, dict) else "" + + +@pytest.mark.parametrize( + "url", + [ + "https://user:token@github.com/psf/requests.git", + "http://someone:hunter2@example.com/x.git", + ], +) +def test_credentials_in_the_url_are_refused(url: str) -> None: + """A submitted URL is persisted and echoed back by the status route. + + So a token handed to this endpoint became readable by anyone who could read + that repository's status. Refused rather than silently stripped — quietly + rewriting what someone submitted is the worse behaviour. + """ + code, message = _submit(url) + assert code == 400 + assert "credentials" in message.lower() + + +def test_a_hostname_resolving_to_a_private_address_is_refused( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The gap this closes: the old check only ever looked at literal IPs. + + `evil.example.com` pointing at cloud metadata or an internal subnet passed + every test, because it is a name and names were not resolved. + """ + + async def resolves_to_metadata(_host: str) -> list[str]: + return ["169.254.169.254"] + + monkeypatch.setattr(repos, "resolve_host", resolves_to_metadata) + code, message = _submit("https://evil.example.com/x.git") + assert code == 400 + assert "private or reserved" in message + + +def test_a_mixed_answer_is_refused_if_any_address_is_private( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """One public answer must not launder a private one.""" + + async def mixed(_host: str) -> list[str]: + return ["140.82.121.4", "10.1.2.3"] + + monkeypatch.setattr(repos, "resolve_host", mixed) + code, _ = _submit("https://mixed.example.com/x.git") + assert code == 400 + + +def test_a_resolution_failure_does_not_reject_the_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A DNS hiccup is not evidence about the host. + + Rejecting here would report an unrelated network problem as an invalid URL, + and the clone would fail later with a clearer reason anyway. + """ + + async def unreachable(_host: str) -> list[str]: + raise OSError("temporary failure in name resolution") + + monkeypatch.setattr(repos, "resolve_host", unreachable) + code, _ = _submit("https://github.com/psf/requests.git") + assert code != 400 + + +def test_the_allowlist_is_the_only_positive_rule(monkeypatch: pytest.MonkeyPatch) -> None: + """Everything else filters what is obviously wrong; this states what is right.""" + from dcode_api.settings import api_settings + + monkeypatch.setattr(api_settings, "repo_url_allowed_hosts", "github.com, gitlab.com") + + code, message = _submit("https://bitbucket.org/team/repo.git") + assert code == 400 + assert "github.com" in message + + assert _submit("https://github.com/psf/requests.git")[0] != 400 + # Subdomains of an allowed host are allowed. + assert _submit("https://raw.github.com/psf/requests.git")[0] != 400 + + +def test_an_empty_allowlist_permits_any_public_host(monkeypatch: pytest.MonkeyPatch) -> None: + from dcode_api.settings import api_settings + + monkeypatch.setattr(api_settings, "repo_url_allowed_hosts", "") + assert _submit("https://bitbucket.org/team/repo.git")[0] != 400 + + +@pytest.mark.parametrize( + "url", + [ + "file:///etc/passwd", + "https://localhost/x.git", + "https://127.0.0.1/x.git", + "https://10.0.0.1/x.git", + "https://[::1]/x.git", + "not-a-git-url", + ], +) +def test_previously_covered_rejections_still_hold(url: str) -> None: + """The literal and scheme filters predate this change and must not regress.""" + assert _submit(url)[0] == 400 + + +def test_an_empty_url_is_refused_by_the_schema_before_the_route() -> None: + """422, not 400, and that is the right layer. + + `min_length=1` on the request body means an empty URL never reaches the + validator — the distinction matters only in that a test asserting 400 here + would be asserting the wrong thing. + """ + assert _submit("")[0] == 422 diff --git a/apps/frontend/.dockerignore b/apps/frontend/.dockerignore deleted file mode 100644 index c6c617a..0000000 --- a/apps/frontend/.dockerignore +++ /dev/null @@ -1,15 +0,0 @@ -# Build-context exclusions for the frontend image. -# -# `npm ci` reinstalls dependencies inside the image, so shipping the local -# tree is pure transfer cost — it measured ~227 MB, and it is also the wrong -# platform's binaries. -node_modules/ -dist/ -.vite/ -coverage/ -*.tsbuildinfo - -.env -.env.* - -.DS_Store diff --git a/apps/worker/src/dcode_worker/settings.py b/apps/worker/src/dcode_worker/settings.py index f94316a..c592e81 100644 --- a/apps/worker/src/dcode_worker/settings.py +++ b/apps/worker/src/dcode_worker/settings.py @@ -12,5 +12,19 @@ class WorkerSettings(SharedSettings): max_file_bytes: int = 1_000_000 # skip .py files larger than this (0 = no cap) max_chunk_chars: int = 20_000 # truncate chunk content beyond this (0 = no cap) + # Cloned workdirs were never removed, for any repository, ever — disk grew + # monotonically with the number of distinct repos anyone ever submitted. On + # a fixed cloud volume that is a slow outage. + # + # This is a cap, not a TTL, and it is OFF by default because eviction is not + # free: the agent's read_file / grep / list_directory tools read that tree at + # query time, so pruning a `ready` repository's workdir degrades its answers + # until it is re-indexed. Search and the call graph are unaffected — they + # read Postgres. A count is the honest knob: an operator can pick a number + # that fits the volume and know exactly what it costs. + # + # 0 keeps the previous behaviour, so no existing deployment changes. + workdir_max_repos: int = 0 + worker_settings = WorkerSettings() diff --git a/apps/worker/src/dcode_worker/stages/clone.py b/apps/worker/src/dcode_worker/stages/clone.py index 12fe7ef..19f5862 100644 --- a/apps/worker/src/dcode_worker/stages/clone.py +++ b/apps/worker/src/dcode_worker/stages/clone.py @@ -1,12 +1,14 @@ """Pipeline stage: shallow-clone the target Git repository.""" import asyncio +import os import shutil from pathlib import Path from uuid import UUID from dcode_worker.context import PipelineContext from dcode_worker.settings import worker_settings +from dcode_worker.stages import prune CLONE_TIMEOUT_SECONDS = 180 @@ -16,10 +18,20 @@ async def run(ctx: PipelineContext) -> PipelineContext: repo_uuid = UUID(ctx.repo_id) workdir = Path(worker_settings.workdir_base).expanduser().resolve() / str(repo_uuid) workdir.parent.mkdir(parents=True, exist_ok=True) + # Reclaim space before needing it. Off unless WORKDIR_MAX_REPOS is set. + prune.prune_workdirs( + workdir.parent, + keep=worker_settings.workdir_max_repos, + protect=str(repo_uuid), + ) if workdir.exists(): shutil.rmtree(workdir) - await _run_git("clone", "--depth=1", ctx.repo_url, str(workdir), cwd=None) + # `--` before the URL. Without it git parses a value beginning with "-" as + # an option, and `--upload-pack=` is command execution. The gateway + # validates the URL, but the worker consumes a queue message and should not + # depend on the only check being upstream of it. + await _run_git("clone", "--depth=1", "--", ctx.repo_url, str(workdir), cwd=None) commit_sha = await _run_git("rev-parse", "HEAD", cwd=workdir) ctx.workdir = str(workdir) @@ -27,11 +39,23 @@ async def run(ctx: PipelineContext) -> PipelineContext: return ctx +# git will sit waiting for a username on a private or mistyped repository, and +# with no terminal it inherits one from the container's environment or hangs +# until the clone timeout. Fail immediately instead: "authentication required" +# is a better answer after two seconds than after three minutes. +_GIT_ENV = { + "GIT_TERMINAL_PROMPT": "0", + "GIT_ASKPASS": "", + "GCM_INTERACTIVE": "never", +} + + async def _run_git(*args: str, cwd: Path | None) -> str: process = await asyncio.create_subprocess_exec( "git", *args, cwd=str(cwd) if cwd is not None else None, + env={**os.environ, **_GIT_ENV}, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) diff --git a/apps/worker/src/dcode_worker/stages/prune.py b/apps/worker/src/dcode_worker/stages/prune.py new file mode 100644 index 0000000..9f58ef7 --- /dev/null +++ b/apps/worker/src/dcode_worker/stages/prune.py @@ -0,0 +1,61 @@ +"""Bound the workdir volume by evicting the least recently used checkouts. + +Runs before a clone, so the space is reclaimed before it is needed rather than +after a disk-full failure. + +Read `WORKDIR_MAX_REPOS` in `dcode_worker.settings` before changing anything +here: eviction is a real trade, not a cleanup. The agent's file tools read these +trees at query time, so an evicted repository keeps working for search and the +call graph and loses `read_file`, `grep` and `list_directory` until it is +re-indexed. +""" + +import logging +import shutil +from pathlib import Path + +from dcode_shared.observability import log_event + +logger = logging.getLogger("dcode.worker.prune") + + +def prune_workdirs(base: Path, *, keep: int, protect: str) -> list[str]: + """Evict oldest-first until at most `keep` workdirs remain. Returns evicted ids. + + `protect` is the repository about to be cloned; evicting it would delete the + directory the caller is on its way to writing. + + `keep <= 0` disables eviction entirely, which is the default. + """ + if keep <= 0 or not base.is_dir(): + return [] + + candidates = [ + path for path in base.iterdir() if path.is_dir() and path.name != protect + ] + # Oldest first. mtime rather than a recorded access time because nothing + # records access: it is a proxy for "least recently indexed", which is the + # closest available signal and is stated here rather than implied. + candidates.sort(key=lambda path: path.stat().st_mtime) + + # `protect` occupies one slot whether or not it exists yet. + excess = (len(candidates) + 1) - keep + evicted: list[str] = [] + for path in candidates[: max(0, excess)]: + try: + shutil.rmtree(path) + except OSError as exc: + # A workdir that will not delete is not a reason to fail the index. + log_event(logger, "workdir_prune_failed", repo_id=path.name, error=str(exc)) + continue + evicted.append(path.name) + + if evicted: + log_event( + logger, + "workdir_pruned", + count=len(evicted), + keep=keep, + repo_ids=",".join(evicted), + ) + return evicted diff --git a/apps/worker/tests/test_prune.py b/apps/worker/tests/test_prune.py new file mode 100644 index 0000000..7d6059a --- /dev/null +++ b/apps/worker/tests/test_prune.py @@ -0,0 +1,96 @@ +"""Workdir retention (F-08). + +Cloned workdirs were never removed, for any repository, ever, so disk grew with +the number of distinct repositories anyone ever submitted. On a fixed cloud +volume that is a slow outage. + +Eviction is not free, which is why it is off by default and why these tests +pin the boundaries rather than just the happy path. +""" + +from pathlib import Path + +from dcode_worker.stages.prune import prune_workdirs + + +def _workdirs(base: Path, names: list[str]) -> None: + for index, name in enumerate(names): + path = base / name + path.mkdir(parents=True) + (path / "file.py").write_text("x", encoding="utf-8") + # Distinct mtimes, oldest first in list order. + import os + + os.utime(path, (1_000_000 + index, 1_000_000 + index)) + + +def test_disabled_by_default(tmp_path: Path) -> None: + """keep=0 keeps the previous behaviour, so no existing deployment changes.""" + _workdirs(tmp_path, ["a", "b", "c"]) + + assert prune_workdirs(tmp_path, keep=0, protect="new") == [] + assert sorted(p.name for p in tmp_path.iterdir()) == ["a", "b", "c"] + + +def test_evicts_oldest_first(tmp_path: Path) -> None: + _workdirs(tmp_path, ["oldest", "middle", "newest"]) + + evicted = prune_workdirs(tmp_path, keep=2, protect="incoming") + + # keep=2 with an incoming repo leaves room for exactly one existing one. + assert evicted == ["oldest", "middle"] + assert sorted(p.name for p in tmp_path.iterdir()) == ["newest"] + + +def test_never_evicts_the_repository_being_cloned(tmp_path: Path) -> None: + """It is the directory the caller is on its way to writing.""" + _workdirs(tmp_path, ["target", "other"]) + + prune_workdirs(tmp_path, keep=1, protect="target") + + assert (tmp_path / "target").is_dir() + assert not (tmp_path / "other").exists() + + +def test_the_protected_repository_occupies_a_slot(tmp_path: Path) -> None: + """Otherwise `keep` is off by one and the cap is quietly exceeded.""" + _workdirs(tmp_path, ["a", "b", "c"]) + + prune_workdirs(tmp_path, keep=3, protect="incoming") + + remaining = sorted(p.name for p in tmp_path.iterdir()) + assert len(remaining) == 2, "two existing + the incoming one is the cap" + + +def test_a_missing_base_directory_is_not_an_error(tmp_path: Path) -> None: + """First boot on an empty volume.""" + assert prune_workdirs(tmp_path / "absent", keep=5, protect="x") == [] + + +def test_a_file_in_the_base_directory_is_ignored(tmp_path: Path) -> None: + _workdirs(tmp_path, ["a"]) + (tmp_path / "stray.txt").write_text("not a workdir", encoding="utf-8") + + prune_workdirs(tmp_path, keep=1, protect="incoming") + + assert (tmp_path / "stray.txt").is_file() + + +def test_an_undeletable_workdir_does_not_fail_the_index( + tmp_path: Path, monkeypatch: object +) -> None: + """Reclaiming space is best-effort; indexing is the job. + + Failing the pipeline because a stale directory would not delete would turn + a housekeeping problem into an outage. + """ + _workdirs(tmp_path, ["stubborn", "fine"]) + + import dcode_worker.stages.prune as prune_module + + def refuse(path: str) -> None: + raise OSError("device or resource busy") + + monkeypatch.setattr(prune_module.shutil, "rmtree", refuse) # type: ignore[attr-defined] + + assert prune_workdirs(tmp_path, keep=1, protect="incoming") == [] diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 9f319cf..e3634b5 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -115,6 +115,10 @@ services: EMBEDDING_TIMEOUT_SECONDS: ${EMBEDDING_TIMEOUT_SECONDS:-300} EMBEDDING_PROVIDER: ${EMBEDDING_PROVIDER:-sidecar} EMBEDDING_API_KEY: ${EMBEDDING_API_KEY:-} + # 0 = never evict, which is the previous behaviour. Set it and the + # oldest checkouts are pruned before a clone — see the setting's + # comment for what an evicted repository loses. + WORKDIR_MAX_REPOS: ${WORKDIR_MAX_REPOS:-0} WORKDIR_BASE: /tmp/dcode-workdirs INTERNAL_API_KEY: ${INTERNAL_API_KEY:?set INTERNAL_API_KEY in .env.production} LOG_LEVEL: ${LOG_LEVEL:-info} @@ -184,8 +188,8 @@ services: frontend: build: - context: ./apps/frontend - dockerfile: ../../infra/docker/frontend.Dockerfile + context: . + dockerfile: infra/docker/frontend.Dockerfile args: VITE_API_BASE_URL: "" ports: diff --git a/docker-compose.yml b/docker-compose.yml index 98ca61f..11ebf68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -121,6 +121,10 @@ services: # never WHICH model runs — both serve jina-embeddings-v2-base-code. EMBEDDING_PROVIDER: ${EMBEDDING_PROVIDER:-sidecar} EMBEDDING_API_KEY: ${EMBEDDING_API_KEY:-} + # 0 = never evict, which is the previous behaviour. Set it and the + # oldest checkouts are pruned before a clone — see the setting's + # comment for what an evicted repository loses. + WORKDIR_MAX_REPOS: ${WORKDIR_MAX_REPOS:-0} WORKDIR_BASE: /tmp/dcode-workdirs INTERNAL_API_KEY: ${INTERNAL_API_KEY:?set INTERNAL_API_KEY in .env} LOG_LEVEL: ${LOG_LEVEL:-info} @@ -229,8 +233,8 @@ services: frontend: build: - context: ./apps/frontend - dockerfile: ../../infra/docker/frontend.Dockerfile + context: . + dockerfile: infra/docker/frontend.Dockerfile args: VITE_API_BASE_URL: ${VITE_API_BASE_URL:-} restart: unless-stopped diff --git a/infra/docker/frontend.Dockerfile b/infra/docker/frontend.Dockerfile index f20892a..edf215e 100644 --- a/infra/docker/frontend.Dockerfile +++ b/infra/docker/frontend.Dockerfile @@ -1,5 +1,9 @@ # Dcode Frontend production image. -# Build context: apps/frontend (per docker-compose). +# +# Build context: the repository ROOT, like every other image here. It used to +# be `apps/frontend`, which worked for compose and made this the only image +# whose context differed — awkward on a platform where the service's config +# file path does not follow its root directory. FROM node:20-alpine AS build WORKDIR /app @@ -7,10 +11,10 @@ WORKDIR /app ARG VITE_API_BASE_URL= ENV VITE_API_BASE_URL=${VITE_API_BASE_URL} -COPY package.json package-lock.json* ./ +COPY apps/frontend/package.json apps/frontend/package-lock.json* ./ RUN npm ci --no-audit --no-fund -COPY . . +COPY apps/frontend/ ./ RUN npm run build FROM nginx:1.27-alpine @@ -22,10 +26,10 @@ FROM nginx:1.27-alpine # substitute nginx's own $host, $remote_addr, $request_uri and $uri, producing # a config that is valid, empty in the places that matter, and broken in a way # that reads like a proxy bug. -COPY nginx.conf.template /etc/nginx/templates/default.conf.template +COPY apps/frontend/nginx.conf.template /etc/nginx/templates/default.conf.template ENV NGINX_ENVSUBST_FILTER="^(PORT|API_UPSTREAM|DNS_RESOLVER)$" -# Defaults reproduce the previous behaviour exactly, so the developer stack and +# Defaults reproduce the compose behaviour, so the developer stack and # docker-compose.prod.yml need no change. Railway overrides PORT (it injects # one), API_UPSTREAM (http://.railway.internal:8000) and DNS_RESOLVER # (its internal resolver) — see Deploy.md R-4. diff --git a/infra/railway/README.md b/infra/railway/README.md new file mode 100644 index 0000000..4a0d327 --- /dev/null +++ b/infra/railway/README.md @@ -0,0 +1,102 @@ +# Railway service configuration + +Three `*.toml` files here, one per service that builds from this repository. +The three data services (Postgres, Redis, RabbitMQ) come from Railway templates +and need no file. + +**Nothing here has been deployed.** These were written from the code, and the +sequence in [`Deploy.md` §10.2](../../Deploy.md) is the intended one rather than +a recorded one. Correct both against what actually happens. + +## Wiring each service + +| Service | Config as Code path | Root Directory | Public domain | +|---|---|---|---| +| `frontend` | `/infra/railway/frontend.toml` | *(empty)* | **yes — the only one** | +| `api` | `/infra/railway/api.toml` | *(empty)* | no | +| `agent-worker` | `/infra/railway/agent-worker.toml` | *(empty)* | no | + +The Config as Code path is **not** relative to Root Directory, so it is absolute +from the repository root. Leave Root Directory empty: every image here builds +from the repository root, including the frontend — it used to build from +`apps/frontend` and was changed precisely so this table has no exception in it. + +`agent-worker` needs **one volume**, mounted at whatever `WORKDIR_BASE` says +(`/tmp/dcode-workdirs` by default). Without it the cloned checkout is lost on +every redeploy, and the agent's `read_file` / `grep` / `list_directory` fail for +every repository until it is re-indexed. Search and the call graph keep working, +because they read Postgres — which is exactly the confusing half-failure the +volume prevents. + +## Variables + +Two rules first, because both fail silently. + +**`EMBEDDING_*` and `RERANKER_*` must agree across `api` and `agent-worker`.** +The API embeds the search query, the worker embeds the corpus, the agent reranks +its own evidence. Configuring one and not the other produces no error — it +produces real vectors in the database being searched with a stub all-zero query +vector (`Deploy.md` §5.4). + +**A `$` in a value is safe in the Railway dashboard** (it is not an env file) but +**not** in a bulk import or a committed env file, where `$VAR` is expanded. A +password hash lost three of its four segments that way and every login failed +forever while the service reported healthy (`Deploy.md` §2.3). The hash format +avoids `$` now; a hand-picked Postgres password still can. + +### `api` + +| Variable | Value | +|---|---| +| `HOST` | `::` — **required.** Railway's private network is IPv6; a service bound to `0.0.0.0` there is reachable by nothing | +| `DATABASE_URL` | `${{Postgres.DATABASE_URL}}`, rewritten to the `postgresql+asyncpg://` scheme | +| `REDIS_URL` | `${{Redis.REDIS_URL}}` | +| `RABBITMQ_URL` | from the RabbitMQ service | +| `AGENT_URL` | `http://agent-worker.railway.internal:8001` | +| `INTERNAL_API_KEY` | 32+ characters. The API refuses to start on the published placeholder | +| `AUTH_ENABLED` | `true` | +| `AUTH_USERNAME`, `AUTH_PASSWORD_HASH`, `AUTH_SESSION_SECRET` | `python -m dcode_api.hash_password` generates the hash and prints a secret command | +| `AUTH_COOKIE_SECURE` | `true` — Railway serves HTTPS | +| `AUTH_DAILY_QUERY_LIMIT` | a number. The login wall stops anonymous callers, not a signed-in loop over three metered APIs | +| `DOCS_ENABLED` | `false` | +| `EMBEDDING_DIM` | `768` for `jina-embeddings-v2-base-code`. **Fixed into the column by the migration; changing it later means a new volume** | +| `EMBEDDING_PROVIDER` / `EMBEDDING_MODEL` / `EMBEDDING_ENDPOINT` / `EMBEDDING_API_KEY` | `jina_api` / `jina-embeddings-v2-base-code` / `https://api.jina.ai/v1` / your key | +| `EMBEDDING_BATCH_SIZE`, `EMBEDDING_MAX_RETRIES`, `EMBEDDING_TIMEOUT_SECONDS` | `32`, `3`, `30`. The defaults are sized for a cold CPU sidecar and would hold one batch for an hour against an API | +| `RERANKER_PROVIDER` / `RERANKER_MODEL` / `RERANKER_ENDPOINT` / `RERANKER_API_KEY` | `siliconflow` / `BAAI/bge-reranker-v2-m3` / `https://api.siliconflow.cn/v1` / your key. **The `.cn` platform** — `.com` serves no BGE and the keys are not interchangeable (`Deploy.md` §2.2) | +| `REPO_URL_ALLOWED_HOSTS` | optional, e.g. `github.com`. The only rule in the URL check that states what is *allowed*, and the available mitigation for DNS rebinding | + +### `agent-worker` + +Same `EMBEDDING_*` / `RERANKER_*` / `INTERNAL_API_KEY` / `DATABASE_URL` / +`REDIS_URL` / `RABBITMQ_URL` values as `api`, plus: + +| Variable | Value | +|---|---| +| `HOST` | `::` | +| `RETRIEVAL_BASE_URL` | `http://api.railway.internal:8000` | +| `WORKDIR_BASE` | the volume's mount path | +| `WORKDIR_MAX_REPOS` | a number the volume can hold. `0` never evicts, which is how the volume fills; read `Deploy.md` §6 PR 5 for what eviction costs | +| `SYNTHESIS_MODEL`, `OPENAI_API_KEY` | `gpt-4o-mini` and your key, to get prose answers rather than the rule-based template | +| `AGENT_REQUEST_BUDGET_SECONDS`, `SSE_HEARTBEAT_SECONDS` | `240`, `20`. Both sized against the platform's request limits (`Deploy.md` R-2) | + +`EMBEDDING_DIM` is needed here too. The agent does not read it directly, but +`dcode_shared.db.models` binds the `chunks.embedding` column type to it at +import time. + +### `frontend` + +| Variable | Value | +|---|---| +| `API_UPSTREAM` | `http://api.railway.internal:8000` — match your actual service name | +| `DNS_RESOLVER` | Railway's internal resolver address | + +`PORT` is injected; do not set it. + +## Order + +Postgres, Redis and RabbitMQ first. Then `api` — its `preDeployCommand` runs the +migration, so `EMBEDDING_DIM` must be right **before** the first deploy of it. +Then `agent-worker`, then `frontend`. + +Then walk the checklist in `Deploy.md` §10. The rows that need a browser need a +browser. diff --git a/infra/railway/agent-worker.toml b/infra/railway/agent-worker.toml new file mode 100644 index 0000000..aa9475f --- /dev/null +++ b/infra/railway/agent-worker.toml @@ -0,0 +1,47 @@ +# Railway service: agent-worker — the LangGraph agent AND the index worker, in +# one container. +# +# Settings → Config as Code: /infra/railway/agent-worker.toml +# Root Directory: empty. No public domain. +# +# WHY THEY SHARE A CONTAINER (Deploy.md R-1): a Railway volume attaches to +# exactly one service, and the worker writes the cloned repository tree that the +# agent's read_file / grep / list_directory tools read back at query time. This +# is a platform workaround, not the architecture — docker-compose.yml still runs +# them as two services against a shared Docker volume. +# +# THE VOLUME: attach one, mounted at the same path as WORKDIR_BASE +# (/tmp/dcode-workdirs unless you change both together). Without it the +# checkout disappears on every redeploy and the agent's file tools fail for +# every repository until it is re-indexed. + +[build] +builder = "DOCKERFILE" +dockerfilePath = "infra/docker/railway-agent-worker.Dockerfile" +watchPatterns = [ + "apps/agent/**", + "apps/worker/**", + "packages/shared/**", + "infra/docker/railway-agent-worker.Dockerfile", + "infra/docker/railway-agent-worker-entrypoint.sh", + "pyproject.toml", + "uv.lock", +] + +[deploy] +# The agent's liveness endpoint. It says nothing about the worker — the +# entrypoint handles that by exiting the container if either process dies, so a +# dead worker becomes a restart rather than a container that looks healthy while +# every submitted repository sits in `queued` forever. +healthcheckPath = "/healthz" +healthcheckTimeout = 120 + +restartPolicyType = "ON_FAILURE" + +# MUST stay 1. Two reasons, either of them sufficient: +# - the volume attaches to one service, and replicas would not share it, so +# each replica's agent would read a different (or empty) checkout; +# - the worker is a single RabbitMQ consumer with prefetch_count=1. A second +# replica is a second consumer competing for the same jobs, and nothing in +# the pipeline is written to be run concurrently on one repository. +numReplicas = 1 diff --git a/infra/railway/api.toml b/infra/railway/api.toml new file mode 100644 index 0000000..3e091b9 --- /dev/null +++ b/infra/railway/api.toml @@ -0,0 +1,46 @@ +# Railway service: api — the FastAPI gateway. +# +# Point the service at this file in Settings → Config as Code, using the path +# from the repository root: /infra/railway/api.toml. That setting does NOT +# follow the service's Root Directory, so it has to be absolute. +# +# Leave Root Directory empty: this image builds from the repository root. +# Do NOT give this service a public domain — the frontend proxies to it over +# private networking, and the whole point of the topology is that the gateway +# is the only backend entry point and is not itself exposed. + +[build] +builder = "DOCKERFILE" +dockerfilePath = "infra/docker/api.Dockerfile" +# Rebuild only when something this image contains actually changed. Without +# these, every commit that touches the frontend or a document redeploys the API. +watchPatterns = [ + "apps/api/**", + "packages/shared/**", + "infra/docker/api.Dockerfile", + "infra/migrations/**", + "pyproject.toml", + "uv.lock", +] + +[deploy] +# Runs before the new deployment takes traffic, inside this container. +# +# This is the right place for it and not a convenience: the migration reads +# EMBEDDING_DIM to size the pgvector column, and EMBEDDING_DIM is configured on +# *this* service (Deploy.md §6 PR 1). Running it anywhere else is how the column +# ends up at a dimension nothing else agrees with. `upgrade head` is idempotent, +# so running it on every deploy is safe. +preDeployCommand = ["uv run alembic -c infra/migrations/alembic.ini upgrade head"] + +# The deep probe, not /healthz. /healthz answers ok with Postgres, Redis and +# RabbitMQ all down; /readyz checks all three plus the embedding-dimension +# agreement, which is the one misconfiguration in this stack that otherwise +# produces no error at all. It skips the dimension check when `chunks` does not +# exist, so it is safe on a first deploy — and preDeployCommand has created the +# table by the time it runs anyway. +healthcheckPath = "/readyz" +healthcheckTimeout = 120 + +restartPolicyType = "ON_FAILURE" +numReplicas = 1 diff --git a/infra/railway/frontend.toml b/infra/railway/frontend.toml new file mode 100644 index 0000000..c5a3ed3 --- /dev/null +++ b/infra/railway/frontend.toml @@ -0,0 +1,37 @@ +# Railway service: frontend — nginx serving the built SPA and proxying /api. +# +# Settings → Config as Code: /infra/railway/frontend.toml +# Root Directory: empty — this image builds from the repository root like the +# others. THIS is the only service that gets a public domain. +# +# Three variables it needs, because the site config is a template rendered at +# boot (Deploy.md R-4): +# +# API_UPSTREAM = http://api.railway.internal:8000 (match your service name) +# DNS_RESOLVER = Railway's internal resolver +# PORT — injected by Railway; do not set it +# +# The proxy target is a variable with a resolver on purpose: Railway's private +# DNS hands out a new address on every redeploy, and nginx caches a literal +# proxy_pass target for the life of the process. Without it the API keeps +# working until the first time *it* is redeployed, then fails with no change on +# this side. + +[build] +builder = "DOCKERFILE" +dockerfilePath = "infra/docker/frontend.Dockerfile" +# results/ and docs/ are in here because the evaluation figures are baked into +# the bundle at build time by scripts/sync_eval_artifacts.py — a new recorded +# run has to reach the deployed page. +watchPatterns = [ + "apps/frontend/**", + "infra/docker/frontend.Dockerfile", + "results/**", + "scripts/sync_eval_artifacts.py", +] + +[deploy] +healthcheckPath = "/healthz" +healthcheckTimeout = 60 +restartPolicyType = "ON_FAILURE" +numReplicas = 1 diff --git a/packages/shared/src/dcode_shared/cache.py b/packages/shared/src/dcode_shared/cache.py index 8c10383..7ae03f7 100644 --- a/packages/shared/src/dcode_shared/cache.py +++ b/packages/shared/src/dcode_shared/cache.py @@ -16,9 +16,26 @@ def embedding_cache_key(model_id: str, text: str) -> str: return f"embed:{model_id}:{digest}" -def tool_cache_key(tool_name: str, repo_id: str, args: dict[str, Any]) -> str: - """`tool:{tool_name}:{repo_id}:{args_hash}` — TTL: 24h.""" - return f"tool:{tool_name}:{repo_id}:{_hash_args(args)}" +def tool_cache_key( + tool_name: str, + repo_id: str, + args: dict[str, Any], + *, + index_revision: int, +) -> str: + """`tool:{tool_name}:{repo_id}:r{index_revision}:{args_hash}` — TTL: 24h. + + `index_revision` is keyword-only and has no default on purpose. It was + absent, and the consequence was invisible: re-indexing a repository left + every cached tool result addressable by the same key for the next 24 hours, + so the agent answered from the previous generation of the corpus. The worker + increments the revision in the same transaction that replaces the chunks + (`dcode_worker.stages.embed`), so including it makes a re-index a cache + miss by construction rather than by remembering to flush. + + A default would have let a new call site reintroduce exactly that. + """ + return f"tool:{tool_name}:{repo_id}:r{index_revision}:{_hash_args(args)}" def query_cache_key( diff --git a/packages/shared/src/dcode_shared/internal.py b/packages/shared/src/dcode_shared/internal.py index d1519fa..0792229 100644 --- a/packages/shared/src/dcode_shared/internal.py +++ b/packages/shared/src/dcode_shared/internal.py @@ -2,6 +2,48 @@ INTERNAL_API_KEY_HEADER = "X-Dcode-Internal-Key" +# The settings default, repeated here so the guard below has something to +# compare against without importing settings into every caller. +PLACEHOLDER_INTERNAL_API_KEY = "__SET_LOCAL_DEV_INTERNAL_API_KEY__" + +# Below this, a key is not a secret. Only enforced where the deployment looks +# like production — see `internal_api_key_error`. +_MIN_PRODUCTION_KEY_CHARS = 32 + + +def internal_api_key_error(key: str, *, strict: bool) -> str | None: + """Why this key cannot protect the internal surface, or None. + + Two rules, and they are deliberately different in strength. + + The placeholder is rejected **always**. It is published in this repository, + so a service running on it has an internal retrieval and graph surface + protected by a value anyone can read. `docker-compose.prod.yml` guards this + with `${INTERNAL_API_KEY:?}`, but that only covers compose — a container + started from the Dockerfile directly, which is what a platform like Railway + does, has no such check. + + The length rule applies only when `strict`. Local development uses a short + readable key on purpose and requiring 32 characters there would be noise + with no reader. `strict` is passed as "this deployment authenticates its + users", which is a proxy for production rather than a fact about it — an + imperfect signal, chosen over having no rule at all outside compose. + """ + stripped = key.strip() + if not stripped: + return "INTERNAL_API_KEY must be set" + if stripped == PLACEHOLDER_INTERNAL_API_KEY: + return ( + "INTERNAL_API_KEY is still the placeholder published in this repository. " + "Set it to a real secret before starting." + ) + if strict and len(stripped) < _MIN_PRODUCTION_KEY_CHARS: + return ( + f"INTERNAL_API_KEY must be at least {_MIN_PRODUCTION_KEY_CHARS} characters " + "when the deployment is gated" + ) + return None + def internal_auth_headers(api_key: str) -> dict[str, str]: """Return the standard internal auth header for private routes.""" diff --git a/packages/shared/src/dcode_shared/schemas.py b/packages/shared/src/dcode_shared/schemas.py index 1cee05a..6feb764 100644 --- a/packages/shared/src/dcode_shared/schemas.py +++ b/packages/shared/src/dcode_shared/schemas.py @@ -68,10 +68,26 @@ class EdgeType(StrEnum): # =========================================================================== +# Request-body bounds. Validation ran on unbounded strings, so a caller could +# make the gateway parse an arbitrarily large body into memory before anything +# rejected it. These are ceilings on abuse, not on legitimate use: no Git URL +# approaches 2 KB, and the gateway already trims history to 4,000 characters +# per turn — the query itself simply had no cap at all. +_MAX_REPO_URL_CHARS = 2048 +_MAX_QUERY_CHARS = 4000 +_MAX_TURN_CHARS = 4000 +_MAX_HISTORY_TURNS = 100 + + class RepoCreateRequest(BaseModel): """POST /api/v1/repos request body.""" - url: str = Field(..., description="Git URL of the repository to index") + url: str = Field( + ..., + min_length=1, + max_length=_MAX_REPO_URL_CHARS, + description="Git URL of the repository to index", + ) class RepoCreateResponse(BaseModel): @@ -145,16 +161,20 @@ class QueryTurn(BaseModel): """ role: Literal["user", "assistant"] - content: str + content: str = Field(..., max_length=_MAX_TURN_CHARS) class QueryRequest(BaseModel): """POST /api/v1/query request body.""" repo_id: UUID - query: str = Field(..., min_length=1) + query: str = Field(..., min_length=1, max_length=_MAX_QUERY_CHARS) + # The gateway trims history to its own turn and character budgets, but that + # happens *after* the body is parsed. This bound is what stops a caller + # sending a hundred thousand turns to be validated first. history: list[QueryTurn] = Field( default_factory=list, + max_length=_MAX_HISTORY_TURNS, description="Prior turns for multi-turn follow-ups (bounded by the gateway).", ) diff --git a/packages/shared/tests/test_schema_roundtrip.py b/packages/shared/tests/test_schema_roundtrip.py index 510ea78..ca848fa 100644 --- a/packages/shared/tests/test_schema_roundtrip.py +++ b/packages/shared/tests/test_schema_roundtrip.py @@ -67,7 +67,9 @@ def test_query_request_rejects_blank_query() -> None: def test_cache_keys_match_design_3_3() -> None: assert embedding_cache_key("bge-code", "hello").startswith("embed:bge-code:") - assert tool_cache_key("search_code", "rid", {"q": "x"}).startswith("tool:search_code:rid:") + assert tool_cache_key("search_code", "rid", {"q": "x"}, index_revision=0).startswith( + "tool:search_code:rid:r0:" + ) assert query_cache_key("rid", "q").startswith("query:rid:") assert job_state_key("rid") == "job:rid"