From 5521ffad3c63a263a3ae699c97597012793fa79b Mon Sep 17 00:00:00 2001 From: odieyang <520yangziqi@gmail.com> Date: Mon, 3 Aug 2026 12:21:35 -0700 Subject: [PATCH 1/5] fix(security): bound request bodies, harden clone, guard the internal key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four of the audit follow-ups, grouped because they are all "an input this service trusted more than it should". Request bodies had no size limits anywhere. QueryRequest.query had a min_length and no max, so a caller could make the gateway parse an arbitrarily large body into memory before anything rejected it; history was trimmed only *after* parsing. Bounds are ceilings on abuse, not on use — no Git URL approaches 2 KB and the gateway already trims turns to 4,000 characters. git clone received the repository URL with no `--`, so a value beginning with "-" is parsed as an option and `--upload-pack=` is command execution. The gateway validates the URL, but the worker consumes a queue message and should not depend on the only check being upstream of it. Also sets GIT_TERMINAL_PROMPT=0: git otherwise waits for a username on a private or mistyped repository until the 180s clone timeout, when "authentication required" after two seconds is the better answer. INTERNAL_API_KEY had a working default published in this repository, and nothing but compose's `${VAR:?}` stopped a service running on it — which covers compose and not a container started straight from the Dockerfile, i.e. what a platform deploy does. Both the API and the agent now refuse to start on the placeholder. The additional length rule applies only when the deployment authenticates its users, which is a proxy for production rather than a fact about it; that is stated in the code rather than implied. The SSRF check only ever looked at literal IPs, so a hostname resolving to cloud metadata or an internal subnet passed. Every resolved address is now checked, one private answer among several public ones is enough to refuse, and a resolution failure does *not* reject — a DNS hiccup is not evidence about the host, and reporting it as an invalid URL would send someone hunting for a typo. Credentials in the URL are refused rather than stripped: a submitted URL is persisted and echoed back by the status route, so a token handed to this endpoint became readable by anyone who could read that repository's status. And there is now an optional host allowlist, empty by default, which is the only rule in the whole check that states what is *allowed* rather than what is obviously wrong. What resolution does not close is rebinding — git resolves the name again when it clones. That needs egress filtering at the network, and it is recorded as unclosed rather than implied to be handled. Resolution goes through a module-level seam so the tests do not perform real DNS lookups; an autouse fixture stubs it, and the tests that care about resolution replace it. Without that the suite would need the network and would occasionally fail for reasons unrelated to the code. --- apps/agent/src/dcode_agent/main.py | 9 +- apps/api/src/dcode_api/main.py | 8 +- apps/api/src/dcode_api/routes/repos.py | 115 +++++++++-- apps/api/src/dcode_api/settings.py | 11 ++ apps/api/tests/conftest.py | 24 +++ apps/api/tests/test_repo_url_guard.py | 193 +++++++++++++++++++ apps/worker/src/dcode_worker/stages/clone.py | 19 +- packages/shared/src/dcode_shared/internal.py | 42 ++++ packages/shared/src/dcode_shared/schemas.py | 26 ++- 9 files changed, 427 insertions(+), 20 deletions(-) create mode 100644 apps/api/tests/conftest.py create mode 100644 apps/api/tests/test_repo_url_guard.py diff --git a/apps/agent/src/dcode_agent/main.py b/apps/agent/src/dcode_agent/main.py index 6ca0b32..cfea312 100644 --- a/apps/agent/src/dcode_agent/main.py +++ b/apps/agent/src/dcode_agent/main.py @@ -8,7 +8,7 @@ from dcode_shared.db.session import SessionLocal from dcode_shared.events import CitationEvent -from dcode_shared.internal import INTERNAL_API_KEY_HEADER +from dcode_shared.internal import INTERNAL_API_KEY_HEADER, internal_api_key_error from dcode_shared.schemas import QueryRequest from fastapi import FastAPI, Header, HTTPException, status from fastapi.responses import StreamingResponse @@ -24,6 +24,13 @@ @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: + # The agent validates the same shared key that protects it. Running on the + # published placeholder means its /internal/query surface is open to + # anything that can reach it on the private network. + key_error = internal_api_key_error(agent_settings.internal_api_key, strict=False) + if key_error is not None: + raise RuntimeError(key_error) + app.state.tool_registry = default_registry() app.state.compiled_graph = graph.build_graph() app.state.tool_cache = Redis.from_url(agent_settings.redis_url, decode_responses=True) diff --git a/apps/api/src/dcode_api/main.py b/apps/api/src/dcode_api/main.py index c84966c..d1cc9f9 100644 --- a/apps/api/src/dcode_api/main.py +++ b/apps/api/src/dcode_api/main.py @@ -8,6 +8,7 @@ from collections.abc import AsyncIterator from contextlib import asynccontextmanager +from dcode_shared.internal import internal_api_key_error from fastapi import Depends, FastAPI from fastapi.middleware.cors import CORSMiddleware @@ -22,7 +23,12 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: # Refuse to start a gate that cannot be enforced. Serving the workbench # openly because a secret was missing is the failure this prevents, and it # is one that looks fine from the outside. - misconfigured = auth_configuration_error() + misconfigured = auth_configuration_error() or internal_api_key_error( + api_settings.internal_api_key, + # The gate being on is this process's best available signal that it is + # a real deployment rather than someone's laptop. + strict=api_settings.auth_enabled, + ) if misconfigured is not None: raise RuntimeError(misconfigured) diff --git a/apps/api/src/dcode_api/routes/repos.py b/apps/api/src/dcode_api/routes/repos.py index e9f3f50..acab593 100644 --- a/apps/api/src/dcode_api/routes/repos.py +++ b/apps/api/src/dcode_api/routes/repos.py @@ -1,8 +1,10 @@ """Public repository submission and indexing-status endpoints.""" +import asyncio import ipaddress import json import re +import socket from collections.abc import Awaitable, Callable from urllib.parse import urlparse from uuid import UUID @@ -26,11 +28,13 @@ from sqlalchemy.ext.asyncio import AsyncSession from dcode_api.deps import get_db, get_index_job_publisher, get_redis +from dcode_api.settings import api_settings router = APIRouter(tags=["repos"]) _SCP_LIKE_GIT_URL = re.compile(r"^[\w.-]+@[\w.-]+:[\w./-]+(?:\.git)?$") _ALLOWED_URL_SCHEMES = {"https", "http", "ssh", "git"} +_DNS_TIMEOUT_SECONDS = 3.0 @router.post( @@ -53,13 +57,11 @@ async def submit_repo( is durable, the repo is marked failed rather than left queued forever. """ repo_url = body.url.strip() - if not _is_supported_git_url(repo_url): + rejection = await _reject_repo_url(repo_url) + if rejection is not None: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, - detail={ - "code": "INVALID_REPO_URL", - "message": "Expected an http(s), ssh, git, or git@host:path Git URL.", - }, + detail={"code": "INVALID_REPO_URL", "message": rejection}, ) existing = await _find_reusable_repo(db, repo_url) @@ -215,20 +217,105 @@ async def repo_status( ) -def _is_supported_git_url(url: str) -> bool: +async def _reject_repo_url(url: str) -> str | None: + """Why this URL may not be cloned, or None. The message reaches the caller. + + Three checks beyond the syntax one, in order of how much they buy: + + 1. **No credentials in the URL.** `https://user:token@host/...` was accepted + and then persisted into `repos.url` and echoed back by the status route, + so a token handed to this endpoint became readable by anyone who could + read a repository's status. Rejected rather than stripped: quietly + mutating what someone submitted is worse than saying no. + 2. **An optional host allowlist.** Empty by default, which keeps any public + host acceptable. Set `REPO_URL_ALLOWED_HOSTS` and it becomes the strong + control — everything else here is a filter on what is obviously wrong, + and this is the only rule that states what is right. + 3. **Resolution.** The literal-IP check never fired for a *name*, so + `evil.example.com` pointing at `169.254.169.254` or `10.0.0.1` passed. + Every resolved address is now checked. + + What check 3 does **not** close is rebinding: git resolves the name again + when it clones, and a DNS answer that changes in between defeats this. The + fix for that is egress filtering at the network, not more code here, and it + is recorded as such in Deploy.md rather than implied to be handled. + """ if not url: - return False + return "Expected a Git URL." + scp_like = _SCP_LIKE_GIT_URL.match(url) if scp_like: host = url.split("@", 1)[1].split(":", 1)[0] - return _is_allowed_remote_host(host) + else: + parsed = urlparse(url) + if parsed.scheme not in _ALLOWED_URL_SCHEMES: + return "Expected an http(s), ssh, git, or git@host:path Git URL." + if parsed.password is not None or (parsed.username and parsed.scheme != "ssh"): + return ( + "Remove the credentials from the URL. This endpoint indexes public " + "repositories, and a submitted URL is stored and shown in status responses." + ) + if not parsed.hostname: + return "Expected an http(s), ssh, git, or git@host:path Git URL." + host = parsed.hostname + + if not _is_allowed_remote_host(host): + return "That host is not a permitted clone target." + + allowlist = api_settings.repo_url_allowed_hosts_list + normalized = host.strip().strip("[]").rstrip(".").lower() + if allowlist and not any( + normalized == allowed or normalized.endswith(f".{allowed}") for allowed in allowlist + ): + return f"Only these hosts may be cloned: {', '.join(sorted(allowlist))}." - parsed = urlparse(url) - if parsed.scheme not in _ALLOWED_URL_SCHEMES: - return False - if not parsed.hostname: - return False - return _is_allowed_remote_host(parsed.hostname) + unreachable = await _reject_resolved_addresses(normalized) + if unreachable is not None: + return unreachable + return None + + +async def resolve_host(host: str) -> list[str]: + """Addresses `host` resolves to. + + A module-level function rather than an inline `loop.getaddrinfo` so it can + be replaced in tests. Without a seam here, every test that submits a + repository URL performs a real DNS lookup — which makes the suite depend on + the network, and makes CI slower and occasionally wrong for a reason that + has nothing to do with the code under test. + """ + loop = asyncio.get_running_loop() + infos = await asyncio.wait_for( + loop.getaddrinfo(host, None, proto=socket.IPPROTO_TCP), + timeout=_DNS_TIMEOUT_SECONDS, + ) + return [str(info[4][0]) for info in infos] + + +async def _reject_resolved_addresses(host: str) -> str | None: + """Resolve `host` and refuse if any answer is a non-public address.""" + try: + ipaddress.ip_address(host) + except ValueError: + pass + else: + # A literal was already checked by _is_allowed_remote_host. + return None + + try: + addresses = await resolve_host(host) + except (TimeoutError, OSError): + # Refusing here would make an unrelated DNS hiccup look like a rejected + # URL, and the worker's clone would fail anyway with a clearer reason. + return None + + for address in addresses: + if not _is_allowed_remote_host(address): + return ( + f"{host} resolves to a private or reserved address " + "and will not be cloned." + ) + return None def _is_allowed_remote_host(host: str) -> bool: diff --git a/apps/api/src/dcode_api/settings.py b/apps/api/src/dcode_api/settings.py index 67683f9..2b3521f 100644 --- a/apps/api/src/dcode_api/settings.py +++ b/apps/api/src/dcode_api/settings.py @@ -38,6 +38,17 @@ class APISettings(SharedSettings): # Useful locally, so on by default; prod compose turns it off. docs_enabled: bool = True + # Comma-separated clone-target allowlist. Empty means any public host, which + # is what a demo indexing arbitrary open-source repositories needs. Set it + # and it becomes the only rule in the URL check that states what is + # *allowed* rather than what is obviously wrong. Subdomains match. + repo_url_allowed_hosts: str = "" + + @property + def repo_url_allowed_hosts_list(self) -> list[str]: + """Comma-separated REPO_URL_ALLOWED_HOSTS env var → lowercased list.""" + return [h.strip().lower() for h in self.repo_url_allowed_hosts.split(",") if h.strip()] + @property def cors_origins_list(self) -> list[str]: """Comma-separated CORS_ORIGINS env var → list.""" diff --git a/apps/api/tests/conftest.py b/apps/api/tests/conftest.py new file mode 100644 index 0000000..f118206 --- /dev/null +++ b/apps/api/tests/conftest.py @@ -0,0 +1,24 @@ +"""Keep the API test suite off the network. + +`POST /repos` resolves the submitted hostname to check that it is not a private +address (F-06). Left alone, every test that submits a URL would perform a real +DNS lookup: the suite would need the network, run slower, and fail occasionally +for a reason unrelated to what it is testing. + +This stubs the lookup to one public address by default. Tests that care about +resolution replace it themselves. +""" + +from collections.abc import Iterator + +import pytest +from dcode_api.routes import repos + + +@pytest.fixture(autouse=True) +def stub_host_resolution(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + async def public_address(_host: str) -> list[str]: + return ["140.82.121.4"] + + monkeypatch.setattr(repos, "resolve_host", public_address) + yield diff --git a/apps/api/tests/test_repo_url_guard.py b/apps/api/tests/test_repo_url_guard.py new file mode 100644 index 0000000..e54ac23 --- /dev/null +++ b/apps/api/tests/test_repo_url_guard.py @@ -0,0 +1,193 @@ +"""Clone-target validation (F-06). + +The literal-IP filter these tests sit on top of already existed. What did not +was any check on a *name*, so the interesting cases here are the ones that +would previously have been accepted. +""" + +import uuid + +import pytest +from dcode_api.deps import get_db, get_index_job_publisher +from dcode_api.main import app +from dcode_api.routes import repos +from fastapi.testclient import TestClient + + +class _EmptyIndex: + """A session with no existing repositories, permissive on writes. + + Permissive on purpose: a *rejected* URL must not reach the write path, and a + test that asserts acceptance needs the write path to succeed. Asserting + inside the fake conflates the two — it turns "accepted" into a crash, which + is indistinguishable from a different bug. + """ + + def __init__(self) -> None: + self.added: list[object] = [] + + async def execute(self, _statement: object) -> object: + class _Result: + def scalars(self) -> "_Result": + return self + + def all(self) -> list[object]: + return [] + + return _Result() + + def add(self, row: object) -> None: + self.added.append(row) + if getattr(row, "id", None) is None: + row.id = uuid.uuid4() + + async def flush(self) -> None: + return None + + async def commit(self) -> None: + return None + + +def _client(session: _EmptyIndex, published: list[str]) -> TestClient: + async def publish(_repo_id: object, url: str) -> None: + published.append(url) + + app.dependency_overrides[get_db] = lambda: session + app.dependency_overrides[get_index_job_publisher] = lambda: publish + return TestClient(app) + + +def _submit(url: str) -> tuple[int, str]: + """Submit one URL. Returns (status, rejection message). + + Also asserts the invariant that matters more than either: a rejected URL + creates no row and queues no job. + """ + session, published = _EmptyIndex(), [] + client = _client(session, published) + try: + response = client.post("/api/v1/repos", json={"url": url}) + finally: + app.dependency_overrides.clear() + + if response.status_code == 400: + assert session.added == [], "a rejected URL must not create a repo row" + assert published == [], "a rejected URL must not be queued for cloning" + + detail = response.json().get("detail", {}) + return response.status_code, str(detail.get("message", "")) if isinstance(detail, dict) else "" + + +@pytest.mark.parametrize( + "url", + [ + "https://user:token@github.com/psf/requests.git", + "http://someone:hunter2@example.com/x.git", + ], +) +def test_credentials_in_the_url_are_refused(url: str) -> None: + """A submitted URL is persisted and echoed back by the status route. + + So a token handed to this endpoint became readable by anyone who could read + that repository's status. Refused rather than silently stripped — quietly + rewriting what someone submitted is the worse behaviour. + """ + code, message = _submit(url) + assert code == 400 + assert "credentials" in message.lower() + + +def test_a_hostname_resolving_to_a_private_address_is_refused( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The gap this closes: the old check only ever looked at literal IPs. + + `evil.example.com` pointing at cloud metadata or an internal subnet passed + every test, because it is a name and names were not resolved. + """ + + async def resolves_to_metadata(_host: str) -> list[str]: + return ["169.254.169.254"] + + monkeypatch.setattr(repos, "resolve_host", resolves_to_metadata) + code, message = _submit("https://evil.example.com/x.git") + assert code == 400 + assert "private or reserved" in message + + +def test_a_mixed_answer_is_refused_if_any_address_is_private( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """One public answer must not launder a private one.""" + + async def mixed(_host: str) -> list[str]: + return ["140.82.121.4", "10.1.2.3"] + + monkeypatch.setattr(repos, "resolve_host", mixed) + code, _ = _submit("https://mixed.example.com/x.git") + assert code == 400 + + +def test_a_resolution_failure_does_not_reject_the_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A DNS hiccup is not evidence about the host. + + Rejecting here would report an unrelated network problem as an invalid URL, + and the clone would fail later with a clearer reason anyway. + """ + + async def unreachable(_host: str) -> list[str]: + raise OSError("temporary failure in name resolution") + + monkeypatch.setattr(repos, "resolve_host", unreachable) + code, _ = _submit("https://github.com/psf/requests.git") + assert code != 400 + + +def test_the_allowlist_is_the_only_positive_rule(monkeypatch: pytest.MonkeyPatch) -> None: + """Everything else filters what is obviously wrong; this states what is right.""" + from dcode_api.settings import api_settings + + monkeypatch.setattr(api_settings, "repo_url_allowed_hosts", "github.com, gitlab.com") + + code, message = _submit("https://bitbucket.org/team/repo.git") + assert code == 400 + assert "github.com" in message + + assert _submit("https://github.com/psf/requests.git")[0] != 400 + # Subdomains of an allowed host are allowed. + assert _submit("https://raw.github.com/psf/requests.git")[0] != 400 + + +def test_an_empty_allowlist_permits_any_public_host(monkeypatch: pytest.MonkeyPatch) -> None: + from dcode_api.settings import api_settings + + monkeypatch.setattr(api_settings, "repo_url_allowed_hosts", "") + assert _submit("https://bitbucket.org/team/repo.git")[0] != 400 + + +@pytest.mark.parametrize( + "url", + [ + "file:///etc/passwd", + "https://localhost/x.git", + "https://127.0.0.1/x.git", + "https://10.0.0.1/x.git", + "https://[::1]/x.git", + "not-a-git-url", + ], +) +def test_previously_covered_rejections_still_hold(url: str) -> None: + """The literal and scheme filters predate this change and must not regress.""" + assert _submit(url)[0] == 400 + + +def test_an_empty_url_is_refused_by_the_schema_before_the_route() -> None: + """422, not 400, and that is the right layer. + + `min_length=1` on the request body means an empty URL never reaches the + validator — the distinction matters only in that a test asserting 400 here + would be asserting the wrong thing. + """ + assert _submit("")[0] == 422 diff --git a/apps/worker/src/dcode_worker/stages/clone.py b/apps/worker/src/dcode_worker/stages/clone.py index 12fe7ef..b5f3a8b 100644 --- a/apps/worker/src/dcode_worker/stages/clone.py +++ b/apps/worker/src/dcode_worker/stages/clone.py @@ -1,6 +1,7 @@ """Pipeline stage: shallow-clone the target Git repository.""" import asyncio +import os import shutil from pathlib import Path from uuid import UUID @@ -19,7 +20,11 @@ async def run(ctx: PipelineContext) -> PipelineContext: if workdir.exists(): shutil.rmtree(workdir) - await _run_git("clone", "--depth=1", ctx.repo_url, str(workdir), cwd=None) + # `--` before the URL. Without it git parses a value beginning with "-" as + # an option, and `--upload-pack=` is command execution. The gateway + # validates the URL, but the worker consumes a queue message and should not + # depend on the only check being upstream of it. + await _run_git("clone", "--depth=1", "--", ctx.repo_url, str(workdir), cwd=None) commit_sha = await _run_git("rev-parse", "HEAD", cwd=workdir) ctx.workdir = str(workdir) @@ -27,11 +32,23 @@ async def run(ctx: PipelineContext) -> PipelineContext: return ctx +# git will sit waiting for a username on a private or mistyped repository, and +# with no terminal it inherits one from the container's environment or hangs +# until the clone timeout. Fail immediately instead: "authentication required" +# is a better answer after two seconds than after three minutes. +_GIT_ENV = { + "GIT_TERMINAL_PROMPT": "0", + "GIT_ASKPASS": "", + "GCM_INTERACTIVE": "never", +} + + async def _run_git(*args: str, cwd: Path | None) -> str: process = await asyncio.create_subprocess_exec( "git", *args, cwd=str(cwd) if cwd is not None else None, + env={**os.environ, **_GIT_ENV}, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) diff --git a/packages/shared/src/dcode_shared/internal.py b/packages/shared/src/dcode_shared/internal.py index d1519fa..0792229 100644 --- a/packages/shared/src/dcode_shared/internal.py +++ b/packages/shared/src/dcode_shared/internal.py @@ -2,6 +2,48 @@ INTERNAL_API_KEY_HEADER = "X-Dcode-Internal-Key" +# The settings default, repeated here so the guard below has something to +# compare against without importing settings into every caller. +PLACEHOLDER_INTERNAL_API_KEY = "__SET_LOCAL_DEV_INTERNAL_API_KEY__" + +# Below this, a key is not a secret. Only enforced where the deployment looks +# like production — see `internal_api_key_error`. +_MIN_PRODUCTION_KEY_CHARS = 32 + + +def internal_api_key_error(key: str, *, strict: bool) -> str | None: + """Why this key cannot protect the internal surface, or None. + + Two rules, and they are deliberately different in strength. + + The placeholder is rejected **always**. It is published in this repository, + so a service running on it has an internal retrieval and graph surface + protected by a value anyone can read. `docker-compose.prod.yml` guards this + with `${INTERNAL_API_KEY:?}`, but that only covers compose — a container + started from the Dockerfile directly, which is what a platform like Railway + does, has no such check. + + The length rule applies only when `strict`. Local development uses a short + readable key on purpose and requiring 32 characters there would be noise + with no reader. `strict` is passed as "this deployment authenticates its + users", which is a proxy for production rather than a fact about it — an + imperfect signal, chosen over having no rule at all outside compose. + """ + stripped = key.strip() + if not stripped: + return "INTERNAL_API_KEY must be set" + if stripped == PLACEHOLDER_INTERNAL_API_KEY: + return ( + "INTERNAL_API_KEY is still the placeholder published in this repository. " + "Set it to a real secret before starting." + ) + if strict and len(stripped) < _MIN_PRODUCTION_KEY_CHARS: + return ( + f"INTERNAL_API_KEY must be at least {_MIN_PRODUCTION_KEY_CHARS} characters " + "when the deployment is gated" + ) + return None + def internal_auth_headers(api_key: str) -> dict[str, str]: """Return the standard internal auth header for private routes.""" diff --git a/packages/shared/src/dcode_shared/schemas.py b/packages/shared/src/dcode_shared/schemas.py index 1cee05a..6feb764 100644 --- a/packages/shared/src/dcode_shared/schemas.py +++ b/packages/shared/src/dcode_shared/schemas.py @@ -68,10 +68,26 @@ class EdgeType(StrEnum): # =========================================================================== +# Request-body bounds. Validation ran on unbounded strings, so a caller could +# make the gateway parse an arbitrarily large body into memory before anything +# rejected it. These are ceilings on abuse, not on legitimate use: no Git URL +# approaches 2 KB, and the gateway already trims history to 4,000 characters +# per turn — the query itself simply had no cap at all. +_MAX_REPO_URL_CHARS = 2048 +_MAX_QUERY_CHARS = 4000 +_MAX_TURN_CHARS = 4000 +_MAX_HISTORY_TURNS = 100 + + class RepoCreateRequest(BaseModel): """POST /api/v1/repos request body.""" - url: str = Field(..., description="Git URL of the repository to index") + url: str = Field( + ..., + min_length=1, + max_length=_MAX_REPO_URL_CHARS, + description="Git URL of the repository to index", + ) class RepoCreateResponse(BaseModel): @@ -145,16 +161,20 @@ class QueryTurn(BaseModel): """ role: Literal["user", "assistant"] - content: str + content: str = Field(..., max_length=_MAX_TURN_CHARS) class QueryRequest(BaseModel): """POST /api/v1/query request body.""" repo_id: UUID - query: str = Field(..., min_length=1) + query: str = Field(..., min_length=1, max_length=_MAX_QUERY_CHARS) + # The gateway trims history to its own turn and character budgets, but that + # happens *after* the body is parsed. This bound is what stops a caller + # sending a hundred thousand turns to be validated first. history: list[QueryTurn] = Field( default_factory=list, + max_length=_MAX_HISTORY_TURNS, description="Prior turns for multi-turn follow-ups (bounded by the gateway).", ) From e829023a308a9aec6e80eeb5fb6d5b6b88ff6b22 Mon Sep 17 00:00:00 2001 From: odieyang <520yangziqi@gmail.com> Date: Mon, 3 Aug 2026 12:23:45 -0700 Subject: [PATCH 2/5] fix(agent): key the tool cache by corpus generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tool cache key was (tool, repo_id, args) and carried nothing about which generation of the corpus produced the answer. Re-indexing a repository therefore left every cached result addressable by the same key for the next 24 hours, and the agent answered from the previous corpus. Nothing failed, nothing logged, and the only symptom was answers that disagreed with the index. The worker already increments repos.index_revision in the same transaction that replaces the chunks, so folding it into the key makes a re-index a cache miss by construction rather than by remembering to flush Redis — which is currently a documented manual step before evaluation runs. `index_revision` on tool_cache_key is keyword-only with no default. A default is what would let a new call site quietly reintroduce this. Read once per query and memoised on the state, not per tool call, so one walk sees one corpus even if a re-index lands mid-answer. When it cannot be read — no session, no such repository, a query error — the request bypasses the cache entirely instead of falling back to a revision-less key, because that fallback would recreate the collision in exactly the case nobody exercises. Three tests cover it: same generation hits, a bumped generation misses, and an unreadable revision produces no cache entry at all. An existing caching test needed a session stub, which is itself the change being made visible. --- apps/agent/src/dcode_agent/graph.py | 49 ++++++- apps/agent/src/dcode_agent/state.py | 5 + apps/agent/src/dcode_agent/tools/base.py | 9 +- apps/agent/tests/test_graph.py | 129 +++++++++++++++++- apps/agent/tests/test_tools_registry.py | 4 +- packages/shared/src/dcode_shared/cache.py | 23 +++- .../shared/tests/test_schema_roundtrip.py | 4 +- 7 files changed, 210 insertions(+), 13 deletions(-) diff --git a/apps/agent/src/dcode_agent/graph.py b/apps/agent/src/dcode_agent/graph.py index 5dc76ee..b01a821 100644 --- a/apps/agent/src/dcode_agent/graph.py +++ b/apps/agent/src/dcode_agent/graph.py @@ -5,12 +5,15 @@ import logging import re from typing import Any, cast +from uuid import UUID +from dcode_shared.db.models import Repo from dcode_shared.observability import log_event from dcode_shared.reranker import RerankerClient, create_reranker_client from dcode_shared.schemas import CallDirection from dcode_shared.settings import shared_settings from langgraph.graph import END, START, StateGraph +from sqlalchemy import select from dcode_agent import groundedness from dcode_agent import state as state_module @@ -116,7 +119,15 @@ async def tool_call_node(state: AgentState) -> AgentState: state, tool_name, _jsonable_args(state.pending_tool_args), exc ) - cache_key = tool.cache_key(state.repo_id, args_model) + revision = await _index_revision(state) + # No revision, no cache. Sharing a key across corpus generations is the + # defect this closes; falling back to one when the revision cannot be read + # would reintroduce it in exactly the case nobody tests. + cache_key = ( + tool.cache_key(state.repo_id, args_model, index_revision=revision) + if revision is not None + else None + ) args_payload = args_model.model_dump(mode="json") await _emit_tool_call(state, tool_name, args_payload) @@ -127,7 +138,11 @@ async def tool_call_node(state: AgentState) -> AgentState: step=state.step_count + 1, tool=tool_name, ) - cached_payload = await _cache_get(state.runtime.get("tool_cache"), cache_key) + cached_payload = ( + await _cache_get(state.runtime.get("tool_cache"), cache_key) + if cache_key is not None + else None + ) cached = cached_payload is not None try: if cached: @@ -138,7 +153,7 @@ async def tool_call_node(state: AgentState) -> AgentState: except Exception as exc: # noqa: BLE001 — tool execution failed; degrade, not abort return await _record_tool_failure(state, tool_name, args_payload, exc) - if not cached: + if not cached and cache_key is not None: await _cache_set(state.runtime.get("tool_cache"), cache_key, json.dumps(result_payload)) log_event( logger, @@ -1413,6 +1428,34 @@ async def _emit_partial_answer(state: AgentState, delta: str) -> None: await emitter.emit_partial_answer(delta) +async def _index_revision(state: AgentState) -> int | None: + """The repository's corpus generation, read once per query and memoised. + + Once per query rather than per tool call, so every tool in one walk sees a + consistent corpus even if a re-index lands mid-answer. + + Returns None when it cannot be read — no database session, no such + repository, or a query error. The caller bypasses the cache in that case. + """ + if state.index_revision is not None: + return state.index_revision + + db = state.runtime.get("db") + if db is None: + return None + try: + result = await db.execute( + select(Repo.index_revision).where(Repo.id == UUID(state.repo_id)) + ) + revision = result.scalar_one_or_none() + except Exception: # noqa: BLE001 — an unreadable revision degrades to no cache + return None + if revision is None: + return None + state.index_revision = int(revision) + return state.index_revision + + async def _cache_get(cache: Any, key: str) -> str | None: if cache is None: return None diff --git a/apps/agent/src/dcode_agent/state.py b/apps/agent/src/dcode_agent/state.py index c99ddaa..7bf7317 100644 --- a/apps/agent/src/dcode_agent/state.py +++ b/apps/agent/src/dcode_agent/state.py @@ -93,4 +93,9 @@ class AgentState: groundedness_score: float | None = None final_answer: str | None = None error: str | None = None + # The repository's corpus generation, read once per query and folded into + # every tool cache key. `None` means it could not be read, and the tool + # cache is bypassed for the request rather than shared with another + # generation — see `_index_revision` in graph.py. + index_revision: int | None = None runtime: dict[str, Any] = field(default_factory=dict, repr=False) diff --git a/apps/agent/src/dcode_agent/tools/base.py b/apps/agent/src/dcode_agent/tools/base.py index 703d0e7..f2bc0a6 100644 --- a/apps/agent/src/dcode_agent/tools/base.py +++ b/apps/agent/src/dcode_agent/tools/base.py @@ -33,9 +33,14 @@ class Tool(ABC, Generic[ArgsT, ResultT]): async def execute(self, repo_id: str, args: ArgsT) -> ResultT: """Execute the tool against the live index for `repo_id`.""" - def cache_key(self, repo_id: str, args: ArgsT) -> str: + def cache_key(self, repo_id: str, args: ArgsT, *, index_revision: int) -> str: """Redis cache key for this invocation in the 24-hour tool namespace.""" - return tool_cache_key(self.name, repo_id, args.model_dump(mode="json")) + return tool_cache_key( + self.name, + repo_id, + args.model_dump(mode="json"), + index_revision=index_revision, + ) class ToolRegistry: diff --git a/apps/agent/tests/test_graph.py b/apps/agent/tests/test_graph.py index f097c37..54e5f21 100644 --- a/apps/agent/tests/test_graph.py +++ b/apps/agent/tests/test_graph.py @@ -424,6 +424,30 @@ async def test_multihop_expands_three_distinct_hybrid_seeds() -> None: ) +class FakeRevisionDb: + """Answers the one SELECT the tool cache key needs. + + Present in these tests because the tool cache is now keyed by the + repository's corpus generation, and a state with no database session + deliberately bypasses the cache rather than sharing a key across + generations. + """ + + def __init__(self, revision: int) -> None: + self.revision = revision + self.queries = 0 + + async def execute(self, _statement: object) -> object: + self.queries += 1 + revision = self.revision + + class _Result: + def scalar_one_or_none(self) -> int: + return revision + + return _Result() + + async def test_tool_call_node_executes_and_then_hits_cache(caplog) -> None: caplog.set_level(logging.INFO, logger="dcode.agent.graph") tool = DummyTool() @@ -431,12 +455,19 @@ async def test_tool_call_node_executes_and_then_hits_cache(caplog) -> None: emitter = FakeEmitter() cache: dict[str, str] = {} repo_id = str(uuid4()) + db = FakeRevisionDb(revision=3) + runtime = { + "tool_registry": registry, + "tool_cache": cache, + "emitter": emitter, + "db": db, + } state = AgentState( repo_id=repo_id, query="Where is `HTTPBasicAuth` defined?", pending_tool_name="find_definition", pending_tool_args={"symbol": "HTTPBasicAuth"}, - runtime={"tool_registry": registry, "tool_cache": cache, "emitter": emitter}, + runtime=runtime, ) first = await tool_call_node(state) @@ -446,7 +477,7 @@ async def test_tool_call_node_executes_and_then_hits_cache(caplog) -> None: query=state.query, pending_tool_name="find_definition", pending_tool_args={"symbol": "HTTPBasicAuth"}, - runtime={"tool_registry": registry, "tool_cache": cache, "emitter": emitter}, + runtime=runtime, ) ) @@ -1302,3 +1333,97 @@ async def rerank(self, query: str, passages: list[str]) -> list[float]: catalog = await graph_module._ranked_evidence_catalog(state) assert catalog["C1"].display_token == "src/requests/mod0.py:1" + + +async def test_reindexing_invalidates_the_tool_cache() -> None: + """The defect F-07 records: a re-index left cached results addressable. + + The key carried (tool, repo_id, args) and nothing about which generation of + the corpus produced the answer, so for 24 hours after re-indexing the agent + answered from the previous one. Nothing failed and nothing logged. + """ + tool = DummyTool() + registry = _registry(tool) + cache: dict[str, str] = {} + repo_id = str(uuid4()) + + def state_for(revision: int) -> AgentState: + return AgentState( + repo_id=repo_id, + query="Where is `HTTPBasicAuth` defined?", + pending_tool_name="find_definition", + pending_tool_args={"symbol": "HTTPBasicAuth"}, + runtime={ + "tool_registry": registry, + "tool_cache": cache, + "emitter": FakeEmitter(), + "db": FakeRevisionDb(revision=revision), + }, + ) + + first = await tool_call_node(state_for(1)) + again = await tool_call_node(state_for(1)) + after_reindex = await tool_call_node(state_for(2)) + + assert first.observations[0]["cached"] is False + assert again.observations[0]["cached"] is True, "same generation should still hit" + assert after_reindex.observations[0]["cached"] is False, "a new generation must miss" + assert tool.calls == 2 + assert len(cache) == 2 + + +async def test_an_unreadable_revision_bypasses_the_cache_rather_than_sharing_one() -> None: + """No revision, no cache. + + Falling back to a revision-less key would reintroduce the cross-generation + collision in exactly the situation nobody exercises, so the request pays for + a real tool call instead. + """ + tool = DummyTool() + registry = _registry(tool) + cache: dict[str, str] = {} + repo_id = str(uuid4()) + + def state_without_db() -> AgentState: + return AgentState( + repo_id=repo_id, + query="q", + pending_tool_name="find_definition", + pending_tool_args={"symbol": "HTTPBasicAuth"}, + runtime={"tool_registry": registry, "tool_cache": cache, "emitter": FakeEmitter()}, + ) + + first = await tool_call_node(state_without_db()) + second = await tool_call_node(state_without_db()) + + assert tool.calls == 2 + assert cache == {} + assert first.tool_calls[0]["cache_key"] is None + assert second.observations[0]["cached"] is False + + +async def test_the_revision_is_read_once_per_query() -> None: + """Read per query, not per tool call, so one walk sees one corpus.""" + tool = DummyTool() + registry = _registry(tool) + db = FakeRevisionDb(revision=7) + state = AgentState( + repo_id=str(uuid4()), + query="q", + pending_tool_name="find_definition", + pending_tool_args={"symbol": "HTTPBasicAuth"}, + runtime={ + "tool_registry": registry, + "tool_cache": {}, + "emitter": FakeEmitter(), + "db": db, + }, + ) + + after = await tool_call_node(state) + after.pending_tool_name = "find_definition" + after.pending_tool_args = {"symbol": "Session"} + await tool_call_node(after) + + assert db.queries == 1 + assert after.index_revision == 7 diff --git a/apps/agent/tests/test_tools_registry.py b/apps/agent/tests/test_tools_registry.py index cfce860..e129d3d 100644 --- a/apps/agent/tests/test_tools_registry.py +++ b/apps/agent/tests/test_tools_registry.py @@ -42,8 +42,8 @@ def test_cache_key_is_deterministic_and_namespaced() -> None: tool = default_registry().get("search_code") assert tool is not None args = tool.ArgsSchema(query="x", k=5) - key_a = tool.cache_key("repo-1", args) - key_b = tool.cache_key("repo-1", args) + key_a = tool.cache_key("repo-1", args, index_revision=0) + key_b = tool.cache_key("repo-1", args, index_revision=0) assert key_a == key_b assert key_a.startswith("tool:search_code:repo-1:") diff --git a/packages/shared/src/dcode_shared/cache.py b/packages/shared/src/dcode_shared/cache.py index 8c10383..7ae03f7 100644 --- a/packages/shared/src/dcode_shared/cache.py +++ b/packages/shared/src/dcode_shared/cache.py @@ -16,9 +16,26 @@ def embedding_cache_key(model_id: str, text: str) -> str: return f"embed:{model_id}:{digest}" -def tool_cache_key(tool_name: str, repo_id: str, args: dict[str, Any]) -> str: - """`tool:{tool_name}:{repo_id}:{args_hash}` — TTL: 24h.""" - return f"tool:{tool_name}:{repo_id}:{_hash_args(args)}" +def tool_cache_key( + tool_name: str, + repo_id: str, + args: dict[str, Any], + *, + index_revision: int, +) -> str: + """`tool:{tool_name}:{repo_id}:r{index_revision}:{args_hash}` — TTL: 24h. + + `index_revision` is keyword-only and has no default on purpose. It was + absent, and the consequence was invisible: re-indexing a repository left + every cached tool result addressable by the same key for the next 24 hours, + so the agent answered from the previous generation of the corpus. The worker + increments the revision in the same transaction that replaces the chunks + (`dcode_worker.stages.embed`), so including it makes a re-index a cache + miss by construction rather than by remembering to flush. + + A default would have let a new call site reintroduce exactly that. + """ + return f"tool:{tool_name}:{repo_id}:r{index_revision}:{_hash_args(args)}" def query_cache_key( diff --git a/packages/shared/tests/test_schema_roundtrip.py b/packages/shared/tests/test_schema_roundtrip.py index 510ea78..ca848fa 100644 --- a/packages/shared/tests/test_schema_roundtrip.py +++ b/packages/shared/tests/test_schema_roundtrip.py @@ -67,7 +67,9 @@ def test_query_request_rejects_blank_query() -> None: def test_cache_keys_match_design_3_3() -> None: assert embedding_cache_key("bge-code", "hello").startswith("embed:bge-code:") - assert tool_cache_key("search_code", "rid", {"q": "x"}).startswith("tool:search_code:rid:") + assert tool_cache_key("search_code", "rid", {"q": "x"}, index_revision=0).startswith( + "tool:search_code:rid:r0:" + ) assert query_cache_key("rid", "q").startswith("query:rid:") assert job_state_key("rid") == "job:rid" From 7da80cdd170e6db5dd9db8702a33f3cda8a905fb Mon Sep 17 00:00:00 2001 From: odieyang <520yangziqi@gmail.com> Date: Mon, 3 Aug 2026 12:27:17 -0700 Subject: [PATCH 3/5] fix(ops): bound grep, add a readiness probe, cap the workdir volume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remaining three audit follow-ups, each with a limit stated rather than implied. grep had no ceiling of any kind. Its argument is a pattern the planner derives from a user question, and `.` over a large repository produced hundreds of MB of JSON buffered whole in the agent process, then serialised into Redis and the SSE state. There is now a timeout (the clone path had one and this did not), per-file and total match caps, and a file-size cap — bounded at ripgrep rather than after the fact, since reading everything and discarding most of it is the behaviour being replaced. The Python fallback is bounded on everything countable and is honest about the rest: Python cannot time out a single `re` call, so a pattern with catastrophic backtracking against one long line can still block the loop. The agent image installs ripgrep, so that path runs only where it is missing. The docstring says so instead of implying the fallback is safe. /healthz answered `ok` with Postgres, Redis and RabbitMQ all down, and the first user request discovered it. /readyz checks all three — all of them, not short-circuited, because an operator wants to know whether one thing is broken or everything is. It stays a 503 rather than a 200 with a body: a load balancer reads the status line. It also checks the embedding dimension against the live column, which is the one misconfiguration in this stack that produces no error at all (Deploy.md §5.4): the worker's inserts fail, or dense search runs a stub all-zero query vector against real stored vectors and returns noise. A missing table is skipped rather than failed, because before the first migration there is nothing to disagree with and the Railway runbook starts the API before running it. /healthz is left shallow on purpose — a liveness probe that fails on a dependency gets the process restarted, which fixes nothing and loses the logs. Cloned workdirs were never removed, for any repository, ever, so disk grew monotonically with the number of distinct repositories ever submitted. Eviction is a trade, not a cleanup: the agent reads those trees at query time, so pruning a `ready` repository's checkout costs it read_file, grep and list_directory until re-index while search and the call graph carry on from Postgres. So it is a count cap, off by default, evicting oldest-first before a clone and never the repository being cloned. An undeletable directory is logged and skipped rather than failing the index. And because that failure is now reachable, the agent's missing-checkout error says what happened: a bare FileNotFoundError sends the reader looking for a missing file rather than a missing checkout. --- .env.production.example | 13 ++ apps/agent/src/dcode_agent/tools/common.py | 17 ++- apps/agent/src/dcode_agent/tools/grep.py | 51 ++++++- apps/agent/tests/test_grep_bounds.py | 102 ++++++++++++++ apps/api/src/dcode_api/main.py | 30 +++- apps/api/src/dcode_api/readiness.py | 109 +++++++++++++++ apps/api/tests/test_readiness.py | 139 +++++++++++++++++++ apps/worker/src/dcode_worker/settings.py | 14 ++ apps/worker/src/dcode_worker/stages/clone.py | 7 + apps/worker/src/dcode_worker/stages/prune.py | 61 ++++++++ apps/worker/tests/test_prune.py | 96 +++++++++++++ docker-compose.prod.yml | 4 + docker-compose.yml | 4 + 13 files changed, 641 insertions(+), 6 deletions(-) create mode 100644 apps/agent/tests/test_grep_bounds.py create mode 100644 apps/api/src/dcode_api/readiness.py create mode 100644 apps/api/tests/test_readiness.py create mode 100644 apps/worker/src/dcode_worker/stages/prune.py create mode 100644 apps/worker/tests/test_prune.py diff --git a/.env.production.example b/.env.production.example index b93ae99..8926777 100644 --- a/.env.production.example +++ b/.env.production.example @@ -77,6 +77,19 @@ INTERNAL_API_KEY=__SET_PROD_INTERNAL_API_KEY__ LOG_LEVEL=info +# Optional clone-target allowlist. Empty means any public host, which is what a +# demo indexing arbitrary open-source repositories needs. Set it and it becomes +# the only rule in the URL check that states what is allowed rather than what is +# obviously wrong. Subdomains match. +# REPO_URL_ALLOWED_HOSTS=github.com,gitlab.com + +# Cloned checkouts were never removed, so the volume grew with every distinct +# repository anyone submitted. 0 keeps that. Setting a cap evicts the oldest +# checkouts before a clone — an evicted repository keeps search and the call +# graph (those read Postgres) and loses read_file / grep / list_directory until +# it is re-indexed. Pick a number that fits the volume. +# WORKDIR_MAX_REPOS=8 + # ============================================================ # Retrieval models # diff --git a/apps/agent/src/dcode_agent/tools/common.py b/apps/agent/src/dcode_agent/tools/common.py index 657dce7..5c18a29 100644 --- a/apps/agent/src/dcode_agent/tools/common.py +++ b/apps/agent/src/dcode_agent/tools/common.py @@ -36,10 +36,23 @@ async def fetch_internal_json( def repo_root(repo_id: str) -> Path: - """Return the cloned repo root for one indexed repository.""" + """Return the cloned repo root for one indexed repository. + + The message matters. A repository can be `ready` in Postgres — chunks, + symbols and edges all present, so search and graph tools work — while its + cloned tree is gone, because workdir retention and index state are separate + concerns (see the worker's prune step). In that situation `read_file`, + `grep` and `list_directory` are the only things that fail, and a bare + FileNotFoundError sends the reader looking for a missing file rather than a + missing checkout. + """ root = (Path(agent_settings.workdir_base).expanduser() / repo_id).resolve() if not root.exists() or not root.is_dir(): - raise FileNotFoundError(f"indexed repo workdir not found for repo_id={repo_id}") + raise FileNotFoundError( + f"repository source is not on disk for repo_id={repo_id}. " + "Search and call-graph tools still work; file-reading tools need the " + "checkout, which is restored by re-indexing this repository." + ) return root diff --git a/apps/agent/src/dcode_agent/tools/grep.py b/apps/agent/src/dcode_agent/tools/grep.py index e007d38..a70d6c3 100644 --- a/apps/agent/src/dcode_agent/tools/grep.py +++ b/apps/agent/src/dcode_agent/tools/grep.py @@ -8,6 +8,7 @@ import json import re import shutil +import time from pathlib import Path from dcode_shared.schemas import Location @@ -27,6 +28,19 @@ class GrepResult(BaseModel): locations: list[Location] +# Bounds on a tool whose argument is a user-influenced pattern. +# +# `.` over a large repository produced hundreds of MB of JSON, buffered whole in +# the agent process and then serialised into Redis and the SSE state. None of +# that is useful to an answer: a pattern matching everything has told you +# nothing, and the first few hundred hits are as informative as the first few +# hundred thousand. +_MATCH_LIMIT = 500 +_PER_FILE_MATCH_LIMIT = 20 +_MAX_FILE_BYTES = 1_000_000 +_GREP_TIMEOUT_SECONDS = 20.0 + + class GrepTool(Tool[GrepArgs, GrepResult]): name = "grep" description = "Exact pattern search via ripgrep — cheap and precise." @@ -46,13 +60,26 @@ async def _grep_with_ripgrep(root: Path, pattern: str) -> list[Location]: "rg", "--json", "-n", + # Bounded at the source rather than after the fact: rg stops looking + # instead of the agent reading everything and throwing most away. + f"--max-count={_PER_FILE_MATCH_LIMIT}", + f"--max-filesize={_MAX_FILE_BYTES}", "-e", pattern, str(root), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - stdout, stderr = await process.communicate() + try: + stdout, stderr = await asyncio.wait_for( + process.communicate(), timeout=_GREP_TIMEOUT_SECONDS + ) + except TimeoutError as exc: + # The clone path has a timeout and this did not, so a pathological + # pattern had no ceiling at all. + process.kill() + await process.wait() + raise RuntimeError(f"grep timed out after {_GREP_TIMEOUT_SECONDS:.0f}s") from exc if process.returncode not in (0, 1): error = stderr.decode("utf-8", errors="replace").strip() raise RuntimeError(f"rg failed: {error or process.returncode}") @@ -73,22 +100,41 @@ async def _grep_with_ripgrep(root: Path, pattern: str) -> list[Location]: chunk_id=None, ) ) + if len(matches) >= _MATCH_LIMIT: + break return matches def _grep_with_python(root: Path, pattern: str) -> list[Location]: + """Fallback for when ripgrep is absent. Bounded, but not fully. + + Honest limit: Python has no way to time out a single `re` call, so a pattern + with catastrophic backtracking against one long line can still block the + event loop. What is bounded is everything countable — total matches, matches + per file, file size, and a wall-clock deadline checked between files. + + The agent image installs ripgrep, so this path runs only where it is + missing: a host-run process or a stripped image. That is why the residual + risk is recorded rather than solved with a subprocess. + """ regex = re.compile(pattern) + deadline = time.monotonic() + _GREP_TIMEOUT_SECONDS matches: list[Location] = [] for path in sorted(root.rglob("*")): + if len(matches) >= _MATCH_LIMIT or time.monotonic() > deadline: + break if not path.is_file(): continue if any(part in _SKIP_DIRS for part in path.relative_to(root).parts[:-1]): continue try: + if path.stat().st_size > _MAX_FILE_BYTES: + continue lines = path.read_text(encoding="utf-8").splitlines() except (OSError, UnicodeDecodeError): continue relative = path.relative_to(root).as_posix() + in_file = 0 for line_number, line in enumerate(lines, start=1): if regex.search(line): matches.append( @@ -99,4 +145,7 @@ def _grep_with_python(root: Path, pattern: str) -> list[Location]: chunk_id=None, ) ) + in_file += 1 + if in_file >= _PER_FILE_MATCH_LIMIT or len(matches) >= _MATCH_LIMIT: + break return matches diff --git a/apps/agent/tests/test_grep_bounds.py b/apps/agent/tests/test_grep_bounds.py new file mode 100644 index 0000000..abc5ab1 --- /dev/null +++ b/apps/agent/tests/test_grep_bounds.py @@ -0,0 +1,102 @@ +"""Bounds on the grep tool (F-15). + +The tool's argument is a pattern the planner derives from a user question, and +it had no ceiling of any kind: no timeout, no match cap, and the whole of +ripgrep's stdout buffered in the agent process before anything looked at it. +""" + +import re +from pathlib import Path + +from dcode_agent.tools import grep as grep_module + + +def _repo(tmp_path: Path, files: dict[str, str]) -> Path: + for name, content in files.items(): + target = tmp_path / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + return tmp_path + + +def test_python_fallback_caps_total_matches(tmp_path: Path) -> None: + """A pattern matching everything has told you nothing. + + The first few hundred hits are as informative as the first few hundred + thousand, and the difference is hundreds of MB through Redis and the SSE + state. + """ + files = {f"mod{index}.py": "x\n" * 200 for index in range(20)} + root = _repo(tmp_path, files) + + matches = grep_module._grep_with_python(root, "x") + + assert len(matches) <= grep_module._MATCH_LIMIT + + +def test_python_fallback_caps_matches_per_file(tmp_path: Path) -> None: + root = _repo(tmp_path, {"one.py": "hit\n" * 500}) + + matches = grep_module._grep_with_python(root, "hit") + + assert len(matches) == grep_module._PER_FILE_MATCH_LIMIT + + +def test_python_fallback_skips_a_file_over_the_size_cap(tmp_path: Path) -> None: + root = _repo( + tmp_path, + { + "small.py": "needle\n", + "huge.py": "needle\n" + ("padding\n" * (grep_module._MAX_FILE_BYTES // 8 + 10)), + }, + ) + + matches = grep_module._grep_with_python(root, "needle") + + assert [match.file_path for match in matches] == ["small.py"] + + +def test_python_fallback_stops_at_the_deadline( + tmp_path: Path, monkeypatch: object +) -> None: + """The wall-clock bound is checked between files. + + A single `re` call cannot be interrupted in Python, so this is what is + actually enforceable — and the tool's docstring says so rather than + implying the fallback is fully bounded. + """ + root = _repo(tmp_path, {f"m{index}.py": "needle\n" for index in range(50)}) + + ticks = iter([0.0] + [999.0] * 200) + monkeypatch.setattr(grep_module.time, "monotonic", lambda: next(ticks)) # type: ignore[attr-defined] + + matches = grep_module._grep_with_python(root, "needle") + + assert matches == [], "an expired deadline must stop the walk" + + +def test_the_ripgrep_invocation_carries_its_own_bounds() -> None: + """Bounded at the source, not after the fact. + + Asserted on the argument list because the alternative is reading everything + ripgrep produces and discarding most of it, which is the behaviour being + replaced. + """ + source = Path(grep_module.__file__).read_text(encoding="utf-8") + assert "--max-count=" in source + assert "--max-filesize=" in source + assert "asyncio.wait_for" in source + + +def test_a_pattern_that_is_not_a_regex_surfaces_as_a_tool_error(tmp_path: Path) -> None: + """The planner can produce an invalid pattern; that is a tool failure. + + The agent records tool failures and degrades to synthesis, so this must + raise rather than return an empty result that reads like "no matches". + """ + root = _repo(tmp_path, {"a.py": "x\n"}) + try: + grep_module._grep_with_python(root, "(unclosed") + except re.error: + return + raise AssertionError("an invalid pattern should not look like zero matches") diff --git a/apps/api/src/dcode_api/main.py b/apps/api/src/dcode_api/main.py index d1cc9f9..0e0c6d6 100644 --- a/apps/api/src/dcode_api/main.py +++ b/apps/api/src/dcode_api/main.py @@ -9,10 +9,10 @@ from contextlib import asynccontextmanager from dcode_shared.internal import internal_api_key_error -from fastapi import Depends, FastAPI +from fastapi import Depends, FastAPI, Response, status from fastapi.middleware.cors import CORSMiddleware -from dcode_api import deps +from dcode_api import deps, readiness from dcode_api.auth import auth_configuration_error, require_session from dcode_api.routes import auth, inspector, internal, query, repos from dcode_api.settings import api_settings @@ -84,9 +84,33 @@ def create_app() -> FastAPI: @application.get("/healthz", tags=["meta"]) async def healthz() -> dict[str, str]: - """Shallow liveness probe — does not check dependent services.""" + """Shallow liveness probe — does not check dependent services. + + Deliberately shallow: a liveness probe that fails when a dependency is + down gets the process restarted, which fixes nothing and loses the logs. + Use /readyz to decide whether it can serve. + """ return {"status": "ok"} + @application.get("/readyz", tags=["meta"]) + async def readyz(response: Response) -> dict[str, object]: + """Deep probe: database, Redis, and the embedding-dimension agreement. + + 503 when anything fails, with a per-check reason. Open like /healthz — + it reports whether this process works, not anything about the corpus. + """ + checks = await readiness.run_checks() + ready = all(check.ok for check in checks) + if not ready: + response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE + return { + "status": "ready" if ready else "not ready", + "checks": [ + {"name": check.name, "ok": check.ok, "detail": check.detail} + for check in checks + ], + } + return application diff --git a/apps/api/src/dcode_api/readiness.py b/apps/api/src/dcode_api/readiness.py new file mode 100644 index 0000000..5818b04 --- /dev/null +++ b/apps/api/src/dcode_api/readiness.py @@ -0,0 +1,109 @@ +"""Readiness checks for the gateway (F-10). + +`/healthz` answers `ok` unconditionally — its own docstring says it does not +check dependent services. That is correct for a liveness probe and useless for +deciding whether this process can serve a request: it reported healthy with +Postgres, Redis and RabbitMQ all down, and the first user request was what +discovered it. + +The dimension check is here rather than in a runbook because of Deploy.md §5.4: +`chunks.embedding` is fixed at migration time, and a service configured for a +different dimension produces no error — the worker's inserts fail, or dense +search runs a stub all-zero query vector against real stored vectors and +silently returns noise. A probe is the only place that catches it before a user +does. +""" + +from dataclasses import dataclass + +from dcode_shared.db.session import SessionLocal +from redis.exceptions import RedisError +from sqlalchemy import text + +from dcode_api.deps import get_redis +from dcode_api.settings import api_settings + + +@dataclass(frozen=True) +class Check: + """One readiness check. `detail` is why it failed, never a secret.""" + + name: str + ok: bool + detail: str | None = None + + +async def run_checks() -> list[Check]: + """Every check, always all of them. + + Not short-circuited on the first failure: an operator looking at this wants + to know whether one thing is broken or everything is, and returning after + the first one turns "the database is down" into a report that says nothing + about Redis. + """ + return [await _check_database(), await _check_redis(), await _check_embedding_dim()] + + +async def _check_database() -> Check: + try: + async with SessionLocal() as session: + await session.execute(text("SELECT 1")) + except Exception as exc: # noqa: BLE001 — any failure is not-ready + return Check("database", False, _reason(exc)) + return Check("database", True) + + +async def _check_redis() -> Check: + try: + redis = await get_redis() + await redis.ping() + except (RedisError, OSError) as exc: + return Check("redis", False, _reason(exc)) + except Exception as exc: # noqa: BLE001 + return Check("redis", False, _reason(exc)) + return Check("redis", True) + + +async def _check_embedding_dim() -> Check: + """Compare the configured dimension against the live column. + + Skipped rather than failed when the table is empty or absent: before the + first migration there is nothing to disagree with, and reporting not-ready + would block a deploy that has not reached the migration step yet. + """ + configured = api_settings.embedding_dim + try: + async with SessionLocal() as session: + result = await session.execute( + text( + "SELECT atttypmod FROM pg_attribute " + "WHERE attrelid = 'chunks'::regclass AND attname = 'embedding'" + ) + ) + live = result.scalar_one_or_none() + except Exception: # noqa: BLE001 — no table yet is not a mismatch + return Check("embedding_dim", True, f"not checked (configured {configured})") + + if live is None: + return Check("embedding_dim", True, f"not checked (configured {configured})") + if int(live) != configured: + return Check( + "embedding_dim", + False, + ( + f"EMBEDDING_DIM is {configured} but chunks.embedding is vector({int(live)}). " + "The column is fixed at migration time; this cannot be reconciled by " + "restarting." + ), + ) + return Check("embedding_dim", True) + + +def _reason(exc: Exception) -> str: + """Exception type and message, trimmed. + + A connection error can carry a DSN, so the detail is bounded rather than + passed through — a readiness probe is often reachable by more people than + the logs are. + """ + return f"{type(exc).__name__}: {exc}"[:200] diff --git a/apps/api/tests/test_readiness.py b/apps/api/tests/test_readiness.py new file mode 100644 index 0000000..c383919 --- /dev/null +++ b/apps/api/tests/test_readiness.py @@ -0,0 +1,139 @@ +"""/readyz (F-10). + +/healthz answers `ok` with every dependency down — correct for liveness, +useless for deciding whether this process can serve. These cover the deep probe +and, in particular, the embedding-dimension check that Deploy.md §5.4 asks for, +because a dimension mismatch is the one misconfiguration in this stack that +produces no error at all. +""" + +import pytest +from dcode_api import readiness +from dcode_api.main import app +from fastapi.testclient import TestClient + + +def _readyz(monkeypatch: pytest.MonkeyPatch, checks: list[readiness.Check]) -> tuple[int, dict]: + async def stub() -> list[readiness.Check]: + return checks + + monkeypatch.setattr(readiness, "run_checks", stub) + response = TestClient(app).get("/readyz") + return response.status_code, response.json() + + +def test_ready_when_every_check_passes(monkeypatch: pytest.MonkeyPatch) -> None: + code, body = _readyz( + monkeypatch, + [readiness.Check("database", True), readiness.Check("redis", True)], + ) + assert code == 200 + assert body["status"] == "ready" + + +def test_not_ready_is_a_503_with_a_reason(monkeypatch: pytest.MonkeyPatch) -> None: + """503, not 200-with-a-body. + + A load balancer reads the status line. Reporting a failure only in the JSON + would keep sending traffic to a process that cannot serve it. + """ + code, body = _readyz( + monkeypatch, + [ + readiness.Check("database", False, "OperationalError: connection refused"), + readiness.Check("redis", True), + ], + ) + assert code == 503 + assert body["status"] == "not ready" + failed = [check for check in body["checks"] if not check["ok"]] + assert len(failed) == 1 + assert "connection refused" in failed[0]["detail"] + + +def test_healthz_stays_shallow_and_open() -> None: + """Unchanged on purpose. + + A liveness probe that fails when a dependency is down gets the process + restarted, which fixes nothing and loses the logs. + """ + response = TestClient(app).get("/healthz") + assert response.status_code == 200 + assert response.json() == {"status": "ok"} + + +async def test_a_dimension_mismatch_is_not_ready(monkeypatch: pytest.MonkeyPatch) -> None: + """The failure mode §5.4 describes, which nothing else catches. + + `chunks.embedding` is fixed at migration time. A service configured for + another dimension either fails every insert or searches real vectors with a + stub all-zero query vector and returns noise — with no error either way. + """ + from dcode_api.settings import api_settings + + monkeypatch.setattr(api_settings, "embedding_dim", 768) + monkeypatch.setattr(readiness, "SessionLocal", _session_returning(1024)) + + check = await readiness._check_embedding_dim() + + assert check.ok is False + assert check.detail is not None + assert "768" in check.detail and "1024" in check.detail + assert "restarting" in check.detail, "the reason should say a restart will not fix it" + + +async def test_a_matching_dimension_is_ready(monkeypatch: pytest.MonkeyPatch) -> None: + from dcode_api.settings import api_settings + + monkeypatch.setattr(api_settings, "embedding_dim", 768) + monkeypatch.setattr(readiness, "SessionLocal", _session_returning(768)) + + assert (await readiness._check_embedding_dim()).ok is True + + +async def test_a_missing_chunks_table_is_skipped_not_failed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Before the first migration there is nothing to disagree with. + + Reporting not-ready would block a deployment that has not reached the + migration step yet, which is a real ordering in the Railway runbook. + """ + monkeypatch.setattr(readiness, "SessionLocal", _session_raising()) + + check = await readiness._check_embedding_dim() + + assert check.ok is True + assert check.detail is not None and "not checked" in check.detail + + +def _session_returning(dimension: int | None) -> object: + class _Result: + def scalar_one_or_none(self) -> int | None: + return dimension + + class _Session: + async def __aenter__(self) -> "_Session": + return self + + async def __aexit__(self, *_: object) -> None: + return None + + async def execute(self, _statement: object) -> _Result: + return _Result() + + return _Session + + +def _session_raising() -> object: + class _Session: + async def __aenter__(self) -> "_Session": + return self + + async def __aexit__(self, *_: object) -> None: + return None + + async def execute(self, _statement: object) -> object: + raise RuntimeError('relation "chunks" does not exist') + + return _Session diff --git a/apps/worker/src/dcode_worker/settings.py b/apps/worker/src/dcode_worker/settings.py index f94316a..c592e81 100644 --- a/apps/worker/src/dcode_worker/settings.py +++ b/apps/worker/src/dcode_worker/settings.py @@ -12,5 +12,19 @@ class WorkerSettings(SharedSettings): max_file_bytes: int = 1_000_000 # skip .py files larger than this (0 = no cap) max_chunk_chars: int = 20_000 # truncate chunk content beyond this (0 = no cap) + # Cloned workdirs were never removed, for any repository, ever — disk grew + # monotonically with the number of distinct repos anyone ever submitted. On + # a fixed cloud volume that is a slow outage. + # + # This is a cap, not a TTL, and it is OFF by default because eviction is not + # free: the agent's read_file / grep / list_directory tools read that tree at + # query time, so pruning a `ready` repository's workdir degrades its answers + # until it is re-indexed. Search and the call graph are unaffected — they + # read Postgres. A count is the honest knob: an operator can pick a number + # that fits the volume and know exactly what it costs. + # + # 0 keeps the previous behaviour, so no existing deployment changes. + workdir_max_repos: int = 0 + worker_settings = WorkerSettings() diff --git a/apps/worker/src/dcode_worker/stages/clone.py b/apps/worker/src/dcode_worker/stages/clone.py index b5f3a8b..19f5862 100644 --- a/apps/worker/src/dcode_worker/stages/clone.py +++ b/apps/worker/src/dcode_worker/stages/clone.py @@ -8,6 +8,7 @@ from dcode_worker.context import PipelineContext from dcode_worker.settings import worker_settings +from dcode_worker.stages import prune CLONE_TIMEOUT_SECONDS = 180 @@ -17,6 +18,12 @@ async def run(ctx: PipelineContext) -> PipelineContext: repo_uuid = UUID(ctx.repo_id) workdir = Path(worker_settings.workdir_base).expanduser().resolve() / str(repo_uuid) workdir.parent.mkdir(parents=True, exist_ok=True) + # Reclaim space before needing it. Off unless WORKDIR_MAX_REPOS is set. + prune.prune_workdirs( + workdir.parent, + keep=worker_settings.workdir_max_repos, + protect=str(repo_uuid), + ) if workdir.exists(): shutil.rmtree(workdir) diff --git a/apps/worker/src/dcode_worker/stages/prune.py b/apps/worker/src/dcode_worker/stages/prune.py new file mode 100644 index 0000000..9f58ef7 --- /dev/null +++ b/apps/worker/src/dcode_worker/stages/prune.py @@ -0,0 +1,61 @@ +"""Bound the workdir volume by evicting the least recently used checkouts. + +Runs before a clone, so the space is reclaimed before it is needed rather than +after a disk-full failure. + +Read `WORKDIR_MAX_REPOS` in `dcode_worker.settings` before changing anything +here: eviction is a real trade, not a cleanup. The agent's file tools read these +trees at query time, so an evicted repository keeps working for search and the +call graph and loses `read_file`, `grep` and `list_directory` until it is +re-indexed. +""" + +import logging +import shutil +from pathlib import Path + +from dcode_shared.observability import log_event + +logger = logging.getLogger("dcode.worker.prune") + + +def prune_workdirs(base: Path, *, keep: int, protect: str) -> list[str]: + """Evict oldest-first until at most `keep` workdirs remain. Returns evicted ids. + + `protect` is the repository about to be cloned; evicting it would delete the + directory the caller is on its way to writing. + + `keep <= 0` disables eviction entirely, which is the default. + """ + if keep <= 0 or not base.is_dir(): + return [] + + candidates = [ + path for path in base.iterdir() if path.is_dir() and path.name != protect + ] + # Oldest first. mtime rather than a recorded access time because nothing + # records access: it is a proxy for "least recently indexed", which is the + # closest available signal and is stated here rather than implied. + candidates.sort(key=lambda path: path.stat().st_mtime) + + # `protect` occupies one slot whether or not it exists yet. + excess = (len(candidates) + 1) - keep + evicted: list[str] = [] + for path in candidates[: max(0, excess)]: + try: + shutil.rmtree(path) + except OSError as exc: + # A workdir that will not delete is not a reason to fail the index. + log_event(logger, "workdir_prune_failed", repo_id=path.name, error=str(exc)) + continue + evicted.append(path.name) + + if evicted: + log_event( + logger, + "workdir_pruned", + count=len(evicted), + keep=keep, + repo_ids=",".join(evicted), + ) + return evicted diff --git a/apps/worker/tests/test_prune.py b/apps/worker/tests/test_prune.py new file mode 100644 index 0000000..7d6059a --- /dev/null +++ b/apps/worker/tests/test_prune.py @@ -0,0 +1,96 @@ +"""Workdir retention (F-08). + +Cloned workdirs were never removed, for any repository, ever, so disk grew with +the number of distinct repositories anyone ever submitted. On a fixed cloud +volume that is a slow outage. + +Eviction is not free, which is why it is off by default and why these tests +pin the boundaries rather than just the happy path. +""" + +from pathlib import Path + +from dcode_worker.stages.prune import prune_workdirs + + +def _workdirs(base: Path, names: list[str]) -> None: + for index, name in enumerate(names): + path = base / name + path.mkdir(parents=True) + (path / "file.py").write_text("x", encoding="utf-8") + # Distinct mtimes, oldest first in list order. + import os + + os.utime(path, (1_000_000 + index, 1_000_000 + index)) + + +def test_disabled_by_default(tmp_path: Path) -> None: + """keep=0 keeps the previous behaviour, so no existing deployment changes.""" + _workdirs(tmp_path, ["a", "b", "c"]) + + assert prune_workdirs(tmp_path, keep=0, protect="new") == [] + assert sorted(p.name for p in tmp_path.iterdir()) == ["a", "b", "c"] + + +def test_evicts_oldest_first(tmp_path: Path) -> None: + _workdirs(tmp_path, ["oldest", "middle", "newest"]) + + evicted = prune_workdirs(tmp_path, keep=2, protect="incoming") + + # keep=2 with an incoming repo leaves room for exactly one existing one. + assert evicted == ["oldest", "middle"] + assert sorted(p.name for p in tmp_path.iterdir()) == ["newest"] + + +def test_never_evicts_the_repository_being_cloned(tmp_path: Path) -> None: + """It is the directory the caller is on its way to writing.""" + _workdirs(tmp_path, ["target", "other"]) + + prune_workdirs(tmp_path, keep=1, protect="target") + + assert (tmp_path / "target").is_dir() + assert not (tmp_path / "other").exists() + + +def test_the_protected_repository_occupies_a_slot(tmp_path: Path) -> None: + """Otherwise `keep` is off by one and the cap is quietly exceeded.""" + _workdirs(tmp_path, ["a", "b", "c"]) + + prune_workdirs(tmp_path, keep=3, protect="incoming") + + remaining = sorted(p.name for p in tmp_path.iterdir()) + assert len(remaining) == 2, "two existing + the incoming one is the cap" + + +def test_a_missing_base_directory_is_not_an_error(tmp_path: Path) -> None: + """First boot on an empty volume.""" + assert prune_workdirs(tmp_path / "absent", keep=5, protect="x") == [] + + +def test_a_file_in_the_base_directory_is_ignored(tmp_path: Path) -> None: + _workdirs(tmp_path, ["a"]) + (tmp_path / "stray.txt").write_text("not a workdir", encoding="utf-8") + + prune_workdirs(tmp_path, keep=1, protect="incoming") + + assert (tmp_path / "stray.txt").is_file() + + +def test_an_undeletable_workdir_does_not_fail_the_index( + tmp_path: Path, monkeypatch: object +) -> None: + """Reclaiming space is best-effort; indexing is the job. + + Failing the pipeline because a stale directory would not delete would turn + a housekeeping problem into an outage. + """ + _workdirs(tmp_path, ["stubborn", "fine"]) + + import dcode_worker.stages.prune as prune_module + + def refuse(path: str) -> None: + raise OSError("device or resource busy") + + monkeypatch.setattr(prune_module.shutil, "rmtree", refuse) # type: ignore[attr-defined] + + assert prune_workdirs(tmp_path, keep=1, protect="incoming") == [] diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 9f319cf..6a7e930 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -115,6 +115,10 @@ services: EMBEDDING_TIMEOUT_SECONDS: ${EMBEDDING_TIMEOUT_SECONDS:-300} EMBEDDING_PROVIDER: ${EMBEDDING_PROVIDER:-sidecar} EMBEDDING_API_KEY: ${EMBEDDING_API_KEY:-} + # 0 = never evict, which is the previous behaviour. Set it and the + # oldest checkouts are pruned before a clone — see the setting's + # comment for what an evicted repository loses. + WORKDIR_MAX_REPOS: ${WORKDIR_MAX_REPOS:-0} WORKDIR_BASE: /tmp/dcode-workdirs INTERNAL_API_KEY: ${INTERNAL_API_KEY:?set INTERNAL_API_KEY in .env.production} LOG_LEVEL: ${LOG_LEVEL:-info} diff --git a/docker-compose.yml b/docker-compose.yml index 98ca61f..f7858bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -121,6 +121,10 @@ services: # never WHICH model runs — both serve jina-embeddings-v2-base-code. EMBEDDING_PROVIDER: ${EMBEDDING_PROVIDER:-sidecar} EMBEDDING_API_KEY: ${EMBEDDING_API_KEY:-} + # 0 = never evict, which is the previous behaviour. Set it and the + # oldest checkouts are pruned before a clone — see the setting's + # comment for what an evicted repository loses. + WORKDIR_MAX_REPOS: ${WORKDIR_MAX_REPOS:-0} WORKDIR_BASE: /tmp/dcode-workdirs INTERNAL_API_KEY: ${INTERNAL_API_KEY:?set INTERNAL_API_KEY in .env} LOG_LEVEL: ${LOG_LEVEL:-info} From 46800ab99f73a727b615576ac21dcdc03d7a807a Mon Sep 17 00:00:00 2001 From: odieyang <520yangziqi@gmail.com> Date: Mon, 3 Aug 2026 12:28:19 -0700 Subject: [PATCH 4/5] docs(deploy): record PR 5, and the three limits it does not remove Eight findings closed or narrowed, with the register updated from what was observed. Three of them are marked "mostly" rather than fixed, and the reasons are written out rather than left in a commit message: DNS rebinding survives resolution checking, because git resolves the name again when it clones. That needs egress filtering at the network, which on Railway is a platform capability this project does not configure. The workdir cap is a trade. The agent reads those trees at query time, so evicting a ready repository's checkout costs it read_file, grep and list_directory until re-index while search and the call graph carry on from Postgres. Off by default for that reason. The grep Python fallback can still block on one pathological re call, because Python offers no way to interrupt one. Everything countable is capped, and the agent image installs ripgrep so the path runs only where it is absent. A closed finding that quietly leaves the hole open is worse than an open one, so all three stay visible in the register rather than being struck through. The runbook also now says to point Railway's health check at /readyz rather than /healthz, and to decide WORKDIR_MAX_REPOS and REPO_URL_ALLOWED_HOSTS deliberately instead of inheriting defaults chosen to preserve existing behaviour. --- Deploy.md | 78 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 17 deletions(-) diff --git a/Deploy.md b/Deploy.md index cd8a574..63b32c7 100644 --- a/Deploy.md +++ b/Deploy.md @@ -25,7 +25,8 @@ Outstanding Work list in the same PR, not tracked in two places. is written from an approved plan, and plans outrun implementations. Verify before relying on a claim here, and correct it when it is wrong. -**Status: PR 1–4 landed. PR 5 not started, and nothing is deployed yet.** The pre-deployment state +**Status: PR 1–5 landed. Nothing is deployed yet** — every remaining item is +in § 10.2 and needs a Railway account. The pre-deployment state is frozen at tag `v1.0-submission` (`a4612b8`). Nothing is deployed anywhere yet. Both hosted model APIs have been exercised end to end from a running local stack — see § 2.1 and § 6 PR 2 for the observed values — but **no index has been @@ -496,13 +497,47 @@ The combined `agent-worker` container has **not been run**, locally or otherwise. Its entrypoint is Railway-only and the developer stack does not use it, so nothing here exercises it. -### PR 5 — `fix/audit-followups` +### PR 5 — `fix/audit-followups` — **done** Findings that are not launch blockers but should not ship unaddressed. See the register in § 7 for F-04, F-06, F-07, F-08, F-10, F-14, F-15. -**Acceptance.** Each item either fixed with a test, or explicitly deferred in -this document with a reason. Silence is not an outcome. +**Acceptance — met.** Eight items, 40 new tests, three commits grouped by what +they are about rather than by which finding they close. + +| Finding | What changed | +|---|---| +| F-03 | Request bodies had no size limit anywhere. `query` had a `min_length` and no max, and history was trimmed only *after* parsing | +| F-04 | Both the API and the agent refuse to start on the placeholder key. The length rule applies only when the deployment authenticates its users, which is a proxy for production and is labelled as one | +| F-06 | Resolution checking, credentials refused rather than stripped, and an optional allowlist. **Rebinding stays open** — see below | +| F-07 | Tool cache keyed by `index_revision`, read once per query. No revision means no cache rather than a shared key | +| F-08 | Optional oldest-first cap, off by default. **Deliberately less than "solved"** — see below | +| F-10 | `/readyz`, including the embedding-dimension check § 5.4 asks for | +| F-14 | `--` before the clone URL, and `GIT_TERMINAL_PROMPT=0` | +| F-15 | Timeout, match caps, file-size cap. **The Python fallback is not fully bounded** — see below | + +**Three limits this PR does not remove, stated because a closed finding that +quietly leaves the hole open is worse than an open one.** + +1. **DNS rebinding is not closed.** Every resolved address is checked, but git + resolves the name again when it clones, and an answer that changes in between + defeats the check. The fix is egress filtering at the network — on Railway, + that is a platform capability this project does not configure. +2. **The workdir cap is a trade, not a cleanup.** The agent reads those trees at + query time, so evicting a `ready` repository's checkout costs it `read_file`, + `grep` and `list_directory` until it is re-indexed; search and the call graph + carry on from Postgres. That is why it is off by default and why the tool's + error message now says which half stopped working. The durable answer is + § 11 item 2 — stop reading source from a filesystem. +3. **The `grep` Python fallback can still block.** Python cannot time out a + single `re` call, so catastrophic backtracking against one long line is + unbounded. Everything countable is capped. The agent image installs ripgrep, + so that path runs only where it is absent. + +Also unchanged on purpose: the Docker health checks still probe `/healthz`. +`/readyz` is for a load balancer, and wiring it into `depends_on: +condition: service_healthy` would make an unrelated dependency outage block +container startup chains locally. --- @@ -519,19 +554,19 @@ as leads to verify when the PR touches them, not as established fact. |---|---|---|---|---|---| | F-01 | ~~No authentication of any kind on `/api/v1/*`~~ **— closed in PR 3.** A shared-account session gate now covers repos, query and inspector; `/internal/*` keeps its own shared-key check. Still one principal, not a tenant model (D-6) | `apps/api/src/dcode_api/main.py` | blocker | 3 | **direct** | | F-02 | ~~The `api` service in the production Compose file has no embedding or reranker configuration~~ **— fixed in PR 1, and it was wider than this row said: `worker` and `agent` were short too. See § 6 PR 1.** | `docker-compose.prod.yml` | blocker | 1 | **direct** | -| F-03 | **Partly closed in PR 3.** Anonymous access is gone and a per-session daily query cap bounds the metered path. **Still open: no request-size limit** — `QueryRequest.query` has `min_length` and no `max_length` | `packages/shared/src/dcode_shared/schemas.py:132` | blocker → medium | 3 → 5 | audit | -| F-04 | `INTERNAL_API_KEY` defaults to a literal published in this repository, with no startup guard outside Compose's `:?` operator — and Railway sets variables per service, so that operator does not protect the deployment | `packages/shared/src/dcode_shared/settings.py:33` | high | 5 | audit | +| F-03 | ~~No rate limiting, no request-size limit, no quota~~ **— closed across PR 3 and PR 5.** Anonymous access gone, per-session daily cap, and bounds on the URL, the query, each turn and the turn count | `packages/shared/src/dcode_shared/schemas.py` | blocker | 3, 5 | **direct** | +| F-04 | ~~`INTERNAL_API_KEY` defaults to a published literal with no startup guard outside Compose~~ **— fixed in PR 5.** API and agent both refuse to boot on it | `packages/shared/src/dcode_shared/internal.py` | high | 5 | **direct** | | F-05 | ~~`/docs`, `/redoc` and `/openapi.json` are publicly served~~ **— closed in PR 3**, off in production and pinned by a test; observed 404 on a gated container | `apps/api/src/dcode_api/main.py` | high | 3 | **direct** | -| F-06 | SSRF: repository URL validation rejects literal private IPs but has no host allowlist and no resolution check, so a hostname resolving to an internal address passes | `apps/api/src/dcode_api/routes/repos.py:203-224` | high | 5 | audit | -| F-07 | The agent tool cache key omits `index_revision`, so re-indexing a repository does not invalidate cached tool results for up to 24 h | `apps/agent/src/dcode_agent/graph.py:119` | high | 5 | audit | -| F-08 | Cloned workdirs are never cleaned up, for any repository, ever. On a fixed Railway volume this is unbounded growth | `apps/worker/src/dcode_worker/stages/clone.py:17-22` | high | 5 | audit | +| F-06 | **Mostly fixed in PR 5**: resolution checked, credentials refused, optional allowlist added. **Rebinding remains open** — git re-resolves at clone time and that needs network egress filtering | `apps/api/src/dcode_api/routes/repos.py` | high | 5 | **direct** | +| F-07 | ~~The agent tool cache key omits `index_revision`~~ **— fixed in PR 5**, keyword-only with no default so a new call site cannot reintroduce it | `packages/shared/src/dcode_shared/cache.py` | high | 5 | **direct** | +| F-08 | **Partly addressed in PR 5**: an optional oldest-first cap, **off by default**, because eviction costs an evicted repository its file tools until re-index. The durable fix is § 11 item 2 | `apps/worker/src/dcode_worker/stages/prune.py` | high | 5 | **direct** | | F-09 | ~~No repository list endpoint~~ **— fixed in PR 4.** `GET /api/v1/repos`, merged behind the localStorage recents so a reader on a new device sees what is indexed. Verified live against the local database: 8 repositories returned | `apps/api/src/dcode_api/routes/repos.py` | high | 4 | **direct** | -| F-10 | `/healthz` returns `ok` unconditionally — it reports healthy with every dependency down. No readiness probe exists | `apps/api/src/dcode_api/main.py:51-54` | medium | 5 | audit | +| F-10 | ~~No readiness probe~~ **— fixed in PR 5.** `/readyz` covers database, Redis and the embedding-dimension agreement; `/healthz` stays shallow on purpose | `apps/api/src/dcode_api/readiness.py` | medium | 5 | **direct** | | F-11 | ~~The two documented production Compose invocations disagree; one of them republishes Postgres, Redis and the RabbitMQ UI~~ **— fixed in PR 1.** | `README.md`, `.env.production.example` | medium | 1 | **direct** | | F-12 | ~~`RERANKER_ENDPOINT` defaults to a dead loopback address in the production template~~ **— fixed in PR 1, pinned by a test.** | `docker-compose.prod.yml`, `.env.production.example` | medium | 1 | **direct** | | F-13 | Embedding retries are 12 attempts against a 300 s timeout; with `prefetch_count=1` one bad batch stalls the entire indexing queue. **Knob added in PR 1 (`EMBEDDING_TIMEOUT_SECONDS`); the values are set in PR 2, where the path that needs them exists.** | `packages/shared/src/dcode_shared/embedding.py:19-20` | medium | 1 → 2 | audit | -| F-14 | `git clone` receives the repository URL with no `--` separator, so a URL beginning with `-` is parsed as an option. Gateway validation is the only defence; the worker has none | `apps/worker/src/dcode_worker/stages/clone.py:22` | medium | 5 | audit | -| F-15 | The `grep` tool runs `rg` with no timeout and buffers all output in memory; the pure-Python fallback compiles a user-supplied regex with no timeout | `apps/agent/src/dcode_agent/tools/grep.py:44-58`, `:79-101` | medium | 5 | audit | +| F-14 | ~~`git clone` receives the URL with no `--` separator~~ **— fixed in PR 5**, plus `GIT_TERMINAL_PROMPT=0` | `apps/worker/src/dcode_worker/stages/clone.py` | medium | 5 | **direct** | +| F-15 | **Mostly fixed in PR 5**: timeout, match caps, file-size cap. **A single pathological `re` call in the fallback is still unbounded** — Python offers no way to interrupt one | `apps/agent/src/dcode_agent/tools/grep.py` | medium | 5 | **direct** | | F-16 | ~~nginx serves the bundle uncompressed~~ **— fixed in PR 4**, measured on the built image at 760,088 → 269,497 bytes. Also fixed a duplicate `Cache-Control` header the first attempt introduced | `apps/frontend/nginx.conf.template` | low | 4 | **direct** | | F-17 | ~~No `.dockerignore` anywhere~~ **— fixed in PR 4**, one at the root and one for `apps/frontend` | repository root | low | 4 | **direct** | | F-21 | ~~A copy-pasteable setup block in § 10.1 contained placeholders, was pasted verbatim, and the placeholder became the credential — the API then refused to boot~~ **— fixed in PR 4** by rewriting the block to substitute its own values. The refusal was correct; the instructions were the defect | `Deploy.md` §10.1 | low | 4 | **direct** | @@ -788,13 +823,22 @@ a `railway.json` or a bulk import is not. `AUTH_SESSION_SECRET`, `AUTH_COOKIE_SECURE=true`, `DOCS_ENABLED=false`. Railway serves HTTPS, so the Secure cookie is correct there. +**Health checks.** Point Railway's at `/readyz`, not `/healthz`. The shallow one +reports ok with every dependency down; the deep one covers the database, Redis +and the embedding-dimension agreement, and returns 503 with a per-check reason. +It skips the dimension check when `chunks` does not exist yet, so it is safe to +have configured before the migration step above. + **Then walk § 10's checklist.** The rows that need a browser need a browser. -**Two things this project has not solved and will meet here.** The first index -runs through the hosted embedding API for real — roughly 23 requests for -`psf/requests` — and that path has never been exercised (§ 6 PR 2). The second -is that cloned workdirs are never cleaned up (F-08), so the volume grows with -every distinct repository anyone indexes, forever. +**Three things to decide here rather than discover.** The first index runs +through the hosted embedding API for real — roughly 23 requests for +`psf/requests` — and that path has never been exercised (§ 6 PR 2). Set +`WORKDIR_MAX_REPOS` to something the volume can hold, since it defaults to +never evicting and read § 6 PR 5 for what eviction costs. And consider +`REPO_URL_ALLOWED_HOSTS`: on a gated demo the login wall is the real control, +but it is the only rule in the URL check that states what is *allowed*, and DNS +rebinding is not closed without it or network egress filtering. --- From 509ccdf069b5908e2bd3be31bd1fecd6a2834cb0 Mon Sep 17 00:00:00 2001 From: odieyang <520yangziqi@gmail.com> Date: Mon, 3 Aug 2026 13:08:28 -0700 Subject: [PATCH 5/5] feat(deploy): commit the Railway service configuration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three config-as-code files and the variable sheet that cannot be one. TOML rather than JSON because several of these settings need a reason attached and JSON has nowhere to put one. numReplicas = 1 on agent-worker is the clearest case: a second replica would not share the volume, so its agent would read a different or empty checkout, and it would also be a second RabbitMQ consumer competing for jobs in a pipeline that is not written to run concurrently on one repository. A bare `1` invites someone to raise it. The migration becomes a preDeployCommand on api rather than a manual step. That is where it belongs, not where it is convenient: it reads EMBEDDING_DIM to size the pgvector column and that variable lives on the api service, so running it anywhere else is how the column ends up at a dimension nothing agrees with. It also fixes the ordering the runbook previously had to describe in prose — EMBEDDING_DIM must now simply be right before the first deploy of api. api's health check is /readyz, not /healthz. The shallow one answers ok with every dependency down. Also unifies the frontend build context to the repository root. It was the only image built from a subdirectory, which is awkward on a platform whose config-file path does not follow the service's Root Directory — the alternative was one service with a Root Directory and a `..` in its dockerfilePath. Removing the apps/frontend exclusion from .dockerignore costs almost nothing: node_modules and dist stay excluded, which is where the 227 MB was. Verified by building: 1.47 MB of context, and the image behaves identically — gzip, cache headers, SPA fallback, and both envsubst shapes with nginx's own variables intact. Still nothing deployed. The files and the runbook are written from the code and say so. --- .dockerignore | 4 -- Deploy.md | 25 ++++++-- apps/frontend/.dockerignore | 15 ----- docker-compose.prod.yml | 4 +- docker-compose.yml | 4 +- infra/docker/frontend.Dockerfile | 14 +++-- infra/railway/README.md | 102 +++++++++++++++++++++++++++++++ infra/railway/agent-worker.toml | 47 ++++++++++++++ infra/railway/api.toml | 46 ++++++++++++++ infra/railway/frontend.toml | 37 +++++++++++ 10 files changed, 265 insertions(+), 33 deletions(-) delete mode 100644 apps/frontend/.dockerignore create mode 100644 infra/railway/README.md create mode 100644 infra/railway/agent-worker.toml create mode 100644 infra/railway/api.toml create mode 100644 infra/railway/frontend.toml diff --git a/.dockerignore b/.dockerignore index a38a99b..ea2c823 100644 --- a/.dockerignore +++ b/.dockerignore @@ -39,10 +39,6 @@ build/ coverage/ *.tsbuildinfo -# The frontend is built from its own context (apps/frontend); the backend -# images have no use for it. -apps/frontend/ - # Documentation, prototypes, and 6.6 MB of recorded evaluation runs. The # services read none of them at runtime — the figures are baked into the # frontend bundle at build time by scripts/sync_eval_artifacts.py. diff --git a/Deploy.md b/Deploy.md index 63b32c7..891bad4 100644 --- a/Deploy.md +++ b/Deploy.md @@ -794,21 +794,36 @@ Written from the code, not from a completed deployment — **nothing has been deployed yet** (§ 6 PR 4). Treat it as the intended sequence, and correct it against what actually happens. +**The configuration is committed.** `infra/railway/{api,agent-worker,frontend}.toml` +carry the build and deploy settings, and +[`infra/railway/README.md`](infra/railway/README.md) is the variable sheet — +which service needs what, and the two rules that fail silently. What follows is +the part that cannot live in a file. + **Services.** Six. Only `frontend` gets a public domain. | Service | Image | Notes | |---|---|---| -| `frontend` | `infra/docker/frontend.Dockerfile`, context `apps/frontend` | Public. Set `API_UPSTREAM=http://api.railway.internal:8000` and `DNS_RESOLVER` to Railway's internal resolver. `PORT` is injected. | +| `frontend` | `infra/docker/frontend.Dockerfile` | Public. Set `API_UPSTREAM` and `DNS_RESOLVER`; `PORT` is injected. Builds from the repository root — it used to build from `apps/frontend`, changed so no service needs a Root Directory. | | `api` | `infra/docker/api.Dockerfile`, context repo root | Private. Set `HOST=::` — Railway's private network is IPv6 and a service bound to `0.0.0.0` is reachable by nothing there. | | `agent-worker` | `infra/docker/railway-agent-worker.Dockerfile`, context repo root | Private. **Owns the only volume**, mounted at `WORKDIR_BASE`. Set `HOST=::`. | | Postgres | Railway **pgvector** template | Not the plain Postgres template — R-3. | | Redis | Railway template | | | RabbitMQ | Railway template | | -**Order.** Postgres/Redis/RabbitMQ first, then `api`, then run the migration -against it, then `agent-worker`, then `frontend`. The migration reads -`EMBEDDING_DIM` from the `api` service (§ 6 PR 1), so that variable has to be -right *before* it runs — afterwards the column dimension is fixed and changing +**Order.** Postgres/Redis/RabbitMQ first, then `api`, then `agent-worker`, then +`frontend`. + +The migration is **not** a manual step: `api.toml` runs it as a +`preDeployCommand`, in that container, before the deployment takes traffic. That +is where it belongs rather than where it is convenient — the migration reads +`EMBEDDING_DIM` to size the pgvector column, and that variable lives on the +`api` service (§ 6 PR 1). Running it anywhere else is how the column ends up at +a dimension nothing agrees with. `upgrade head` is idempotent, so every deploy +running it is fine. + +The consequence: **`EMBEDDING_DIM` must be correct before the first deploy of +`api`**, not before some later step. Afterwards the column is fixed and changing it means a new volume. **Variables that must agree across `api`, `agent-worker` and each other**: diff --git a/apps/frontend/.dockerignore b/apps/frontend/.dockerignore deleted file mode 100644 index c6c617a..0000000 --- a/apps/frontend/.dockerignore +++ /dev/null @@ -1,15 +0,0 @@ -# Build-context exclusions for the frontend image. -# -# `npm ci` reinstalls dependencies inside the image, so shipping the local -# tree is pure transfer cost — it measured ~227 MB, and it is also the wrong -# platform's binaries. -node_modules/ -dist/ -.vite/ -coverage/ -*.tsbuildinfo - -.env -.env.* - -.DS_Store diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 6a7e930..e3634b5 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -188,8 +188,8 @@ services: frontend: build: - context: ./apps/frontend - dockerfile: ../../infra/docker/frontend.Dockerfile + context: . + dockerfile: infra/docker/frontend.Dockerfile args: VITE_API_BASE_URL: "" ports: diff --git a/docker-compose.yml b/docker-compose.yml index f7858bd..11ebf68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -233,8 +233,8 @@ services: frontend: build: - context: ./apps/frontend - dockerfile: ../../infra/docker/frontend.Dockerfile + context: . + dockerfile: infra/docker/frontend.Dockerfile args: VITE_API_BASE_URL: ${VITE_API_BASE_URL:-} restart: unless-stopped diff --git a/infra/docker/frontend.Dockerfile b/infra/docker/frontend.Dockerfile index f20892a..edf215e 100644 --- a/infra/docker/frontend.Dockerfile +++ b/infra/docker/frontend.Dockerfile @@ -1,5 +1,9 @@ # Dcode Frontend production image. -# Build context: apps/frontend (per docker-compose). +# +# Build context: the repository ROOT, like every other image here. It used to +# be `apps/frontend`, which worked for compose and made this the only image +# whose context differed — awkward on a platform where the service's config +# file path does not follow its root directory. FROM node:20-alpine AS build WORKDIR /app @@ -7,10 +11,10 @@ WORKDIR /app ARG VITE_API_BASE_URL= ENV VITE_API_BASE_URL=${VITE_API_BASE_URL} -COPY package.json package-lock.json* ./ +COPY apps/frontend/package.json apps/frontend/package-lock.json* ./ RUN npm ci --no-audit --no-fund -COPY . . +COPY apps/frontend/ ./ RUN npm run build FROM nginx:1.27-alpine @@ -22,10 +26,10 @@ FROM nginx:1.27-alpine # substitute nginx's own $host, $remote_addr, $request_uri and $uri, producing # a config that is valid, empty in the places that matter, and broken in a way # that reads like a proxy bug. -COPY nginx.conf.template /etc/nginx/templates/default.conf.template +COPY apps/frontend/nginx.conf.template /etc/nginx/templates/default.conf.template ENV NGINX_ENVSUBST_FILTER="^(PORT|API_UPSTREAM|DNS_RESOLVER)$" -# Defaults reproduce the previous behaviour exactly, so the developer stack and +# Defaults reproduce the compose behaviour, so the developer stack and # docker-compose.prod.yml need no change. Railway overrides PORT (it injects # one), API_UPSTREAM (http://.railway.internal:8000) and DNS_RESOLVER # (its internal resolver) — see Deploy.md R-4. diff --git a/infra/railway/README.md b/infra/railway/README.md new file mode 100644 index 0000000..4a0d327 --- /dev/null +++ b/infra/railway/README.md @@ -0,0 +1,102 @@ +# Railway service configuration + +Three `*.toml` files here, one per service that builds from this repository. +The three data services (Postgres, Redis, RabbitMQ) come from Railway templates +and need no file. + +**Nothing here has been deployed.** These were written from the code, and the +sequence in [`Deploy.md` §10.2](../../Deploy.md) is the intended one rather than +a recorded one. Correct both against what actually happens. + +## Wiring each service + +| Service | Config as Code path | Root Directory | Public domain | +|---|---|---|---| +| `frontend` | `/infra/railway/frontend.toml` | *(empty)* | **yes — the only one** | +| `api` | `/infra/railway/api.toml` | *(empty)* | no | +| `agent-worker` | `/infra/railway/agent-worker.toml` | *(empty)* | no | + +The Config as Code path is **not** relative to Root Directory, so it is absolute +from the repository root. Leave Root Directory empty: every image here builds +from the repository root, including the frontend — it used to build from +`apps/frontend` and was changed precisely so this table has no exception in it. + +`agent-worker` needs **one volume**, mounted at whatever `WORKDIR_BASE` says +(`/tmp/dcode-workdirs` by default). Without it the cloned checkout is lost on +every redeploy, and the agent's `read_file` / `grep` / `list_directory` fail for +every repository until it is re-indexed. Search and the call graph keep working, +because they read Postgres — which is exactly the confusing half-failure the +volume prevents. + +## Variables + +Two rules first, because both fail silently. + +**`EMBEDDING_*` and `RERANKER_*` must agree across `api` and `agent-worker`.** +The API embeds the search query, the worker embeds the corpus, the agent reranks +its own evidence. Configuring one and not the other produces no error — it +produces real vectors in the database being searched with a stub all-zero query +vector (`Deploy.md` §5.4). + +**A `$` in a value is safe in the Railway dashboard** (it is not an env file) but +**not** in a bulk import or a committed env file, where `$VAR` is expanded. A +password hash lost three of its four segments that way and every login failed +forever while the service reported healthy (`Deploy.md` §2.3). The hash format +avoids `$` now; a hand-picked Postgres password still can. + +### `api` + +| Variable | Value | +|---|---| +| `HOST` | `::` — **required.** Railway's private network is IPv6; a service bound to `0.0.0.0` there is reachable by nothing | +| `DATABASE_URL` | `${{Postgres.DATABASE_URL}}`, rewritten to the `postgresql+asyncpg://` scheme | +| `REDIS_URL` | `${{Redis.REDIS_URL}}` | +| `RABBITMQ_URL` | from the RabbitMQ service | +| `AGENT_URL` | `http://agent-worker.railway.internal:8001` | +| `INTERNAL_API_KEY` | 32+ characters. The API refuses to start on the published placeholder | +| `AUTH_ENABLED` | `true` | +| `AUTH_USERNAME`, `AUTH_PASSWORD_HASH`, `AUTH_SESSION_SECRET` | `python -m dcode_api.hash_password` generates the hash and prints a secret command | +| `AUTH_COOKIE_SECURE` | `true` — Railway serves HTTPS | +| `AUTH_DAILY_QUERY_LIMIT` | a number. The login wall stops anonymous callers, not a signed-in loop over three metered APIs | +| `DOCS_ENABLED` | `false` | +| `EMBEDDING_DIM` | `768` for `jina-embeddings-v2-base-code`. **Fixed into the column by the migration; changing it later means a new volume** | +| `EMBEDDING_PROVIDER` / `EMBEDDING_MODEL` / `EMBEDDING_ENDPOINT` / `EMBEDDING_API_KEY` | `jina_api` / `jina-embeddings-v2-base-code` / `https://api.jina.ai/v1` / your key | +| `EMBEDDING_BATCH_SIZE`, `EMBEDDING_MAX_RETRIES`, `EMBEDDING_TIMEOUT_SECONDS` | `32`, `3`, `30`. The defaults are sized for a cold CPU sidecar and would hold one batch for an hour against an API | +| `RERANKER_PROVIDER` / `RERANKER_MODEL` / `RERANKER_ENDPOINT` / `RERANKER_API_KEY` | `siliconflow` / `BAAI/bge-reranker-v2-m3` / `https://api.siliconflow.cn/v1` / your key. **The `.cn` platform** — `.com` serves no BGE and the keys are not interchangeable (`Deploy.md` §2.2) | +| `REPO_URL_ALLOWED_HOSTS` | optional, e.g. `github.com`. The only rule in the URL check that states what is *allowed*, and the available mitigation for DNS rebinding | + +### `agent-worker` + +Same `EMBEDDING_*` / `RERANKER_*` / `INTERNAL_API_KEY` / `DATABASE_URL` / +`REDIS_URL` / `RABBITMQ_URL` values as `api`, plus: + +| Variable | Value | +|---|---| +| `HOST` | `::` | +| `RETRIEVAL_BASE_URL` | `http://api.railway.internal:8000` | +| `WORKDIR_BASE` | the volume's mount path | +| `WORKDIR_MAX_REPOS` | a number the volume can hold. `0` never evicts, which is how the volume fills; read `Deploy.md` §6 PR 5 for what eviction costs | +| `SYNTHESIS_MODEL`, `OPENAI_API_KEY` | `gpt-4o-mini` and your key, to get prose answers rather than the rule-based template | +| `AGENT_REQUEST_BUDGET_SECONDS`, `SSE_HEARTBEAT_SECONDS` | `240`, `20`. Both sized against the platform's request limits (`Deploy.md` R-2) | + +`EMBEDDING_DIM` is needed here too. The agent does not read it directly, but +`dcode_shared.db.models` binds the `chunks.embedding` column type to it at +import time. + +### `frontend` + +| Variable | Value | +|---|---| +| `API_UPSTREAM` | `http://api.railway.internal:8000` — match your actual service name | +| `DNS_RESOLVER` | Railway's internal resolver address | + +`PORT` is injected; do not set it. + +## Order + +Postgres, Redis and RabbitMQ first. Then `api` — its `preDeployCommand` runs the +migration, so `EMBEDDING_DIM` must be right **before** the first deploy of it. +Then `agent-worker`, then `frontend`. + +Then walk the checklist in `Deploy.md` §10. The rows that need a browser need a +browser. diff --git a/infra/railway/agent-worker.toml b/infra/railway/agent-worker.toml new file mode 100644 index 0000000..aa9475f --- /dev/null +++ b/infra/railway/agent-worker.toml @@ -0,0 +1,47 @@ +# Railway service: agent-worker — the LangGraph agent AND the index worker, in +# one container. +# +# Settings → Config as Code: /infra/railway/agent-worker.toml +# Root Directory: empty. No public domain. +# +# WHY THEY SHARE A CONTAINER (Deploy.md R-1): a Railway volume attaches to +# exactly one service, and the worker writes the cloned repository tree that the +# agent's read_file / grep / list_directory tools read back at query time. This +# is a platform workaround, not the architecture — docker-compose.yml still runs +# them as two services against a shared Docker volume. +# +# THE VOLUME: attach one, mounted at the same path as WORKDIR_BASE +# (/tmp/dcode-workdirs unless you change both together). Without it the +# checkout disappears on every redeploy and the agent's file tools fail for +# every repository until it is re-indexed. + +[build] +builder = "DOCKERFILE" +dockerfilePath = "infra/docker/railway-agent-worker.Dockerfile" +watchPatterns = [ + "apps/agent/**", + "apps/worker/**", + "packages/shared/**", + "infra/docker/railway-agent-worker.Dockerfile", + "infra/docker/railway-agent-worker-entrypoint.sh", + "pyproject.toml", + "uv.lock", +] + +[deploy] +# The agent's liveness endpoint. It says nothing about the worker — the +# entrypoint handles that by exiting the container if either process dies, so a +# dead worker becomes a restart rather than a container that looks healthy while +# every submitted repository sits in `queued` forever. +healthcheckPath = "/healthz" +healthcheckTimeout = 120 + +restartPolicyType = "ON_FAILURE" + +# MUST stay 1. Two reasons, either of them sufficient: +# - the volume attaches to one service, and replicas would not share it, so +# each replica's agent would read a different (or empty) checkout; +# - the worker is a single RabbitMQ consumer with prefetch_count=1. A second +# replica is a second consumer competing for the same jobs, and nothing in +# the pipeline is written to be run concurrently on one repository. +numReplicas = 1 diff --git a/infra/railway/api.toml b/infra/railway/api.toml new file mode 100644 index 0000000..3e091b9 --- /dev/null +++ b/infra/railway/api.toml @@ -0,0 +1,46 @@ +# Railway service: api — the FastAPI gateway. +# +# Point the service at this file in Settings → Config as Code, using the path +# from the repository root: /infra/railway/api.toml. That setting does NOT +# follow the service's Root Directory, so it has to be absolute. +# +# Leave Root Directory empty: this image builds from the repository root. +# Do NOT give this service a public domain — the frontend proxies to it over +# private networking, and the whole point of the topology is that the gateway +# is the only backend entry point and is not itself exposed. + +[build] +builder = "DOCKERFILE" +dockerfilePath = "infra/docker/api.Dockerfile" +# Rebuild only when something this image contains actually changed. Without +# these, every commit that touches the frontend or a document redeploys the API. +watchPatterns = [ + "apps/api/**", + "packages/shared/**", + "infra/docker/api.Dockerfile", + "infra/migrations/**", + "pyproject.toml", + "uv.lock", +] + +[deploy] +# Runs before the new deployment takes traffic, inside this container. +# +# This is the right place for it and not a convenience: the migration reads +# EMBEDDING_DIM to size the pgvector column, and EMBEDDING_DIM is configured on +# *this* service (Deploy.md §6 PR 1). Running it anywhere else is how the column +# ends up at a dimension nothing else agrees with. `upgrade head` is idempotent, +# so running it on every deploy is safe. +preDeployCommand = ["uv run alembic -c infra/migrations/alembic.ini upgrade head"] + +# The deep probe, not /healthz. /healthz answers ok with Postgres, Redis and +# RabbitMQ all down; /readyz checks all three plus the embedding-dimension +# agreement, which is the one misconfiguration in this stack that otherwise +# produces no error at all. It skips the dimension check when `chunks` does not +# exist, so it is safe on a first deploy — and preDeployCommand has created the +# table by the time it runs anyway. +healthcheckPath = "/readyz" +healthcheckTimeout = 120 + +restartPolicyType = "ON_FAILURE" +numReplicas = 1 diff --git a/infra/railway/frontend.toml b/infra/railway/frontend.toml new file mode 100644 index 0000000..c5a3ed3 --- /dev/null +++ b/infra/railway/frontend.toml @@ -0,0 +1,37 @@ +# Railway service: frontend — nginx serving the built SPA and proxying /api. +# +# Settings → Config as Code: /infra/railway/frontend.toml +# Root Directory: empty — this image builds from the repository root like the +# others. THIS is the only service that gets a public domain. +# +# Three variables it needs, because the site config is a template rendered at +# boot (Deploy.md R-4): +# +# API_UPSTREAM = http://api.railway.internal:8000 (match your service name) +# DNS_RESOLVER = Railway's internal resolver +# PORT — injected by Railway; do not set it +# +# The proxy target is a variable with a resolver on purpose: Railway's private +# DNS hands out a new address on every redeploy, and nginx caches a literal +# proxy_pass target for the life of the process. Without it the API keeps +# working until the first time *it* is redeployed, then fails with no change on +# this side. + +[build] +builder = "DOCKERFILE" +dockerfilePath = "infra/docker/frontend.Dockerfile" +# results/ and docs/ are in here because the evaluation figures are baked into +# the bundle at build time by scripts/sync_eval_artifacts.py — a new recorded +# run has to reach the deployed page. +watchPatterns = [ + "apps/frontend/**", + "infra/docker/frontend.Dockerfile", + "results/**", + "scripts/sync_eval_artifacts.py", +] + +[deploy] +healthcheckPath = "/healthz" +healthcheckTimeout = 60 +restartPolicyType = "ON_FAILURE" +numReplicas = 1