From 02ed9aaaaaf12b115c2825b3e5b5e8e77fc9257d Mon Sep 17 00:00:00 2001 From: odieyang <520yangziqi@gmail.com> Date: Mon, 3 Aug 2026 13:47:26 -0700 Subject: [PATCH] fix(deploy): make the images self-contained instead of installing at start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first Railway deploy of `api` spent twelve minutes and then failed with `error: Failed to spawn: alembic`. Two causes, both invisible locally. `uv sync --no-dev` at the workspace root syncs the ROOT project, not the members. The root pyproject declares no dependencies, so that build step installed nothing — the local images have two entries in site-packages and no alembic. Everything was really being installed by `uv run` at container start, every start. `--package dcode-api` (and the equivalent for worker and agent) makes the build actually produce a usable environment. Then `uv run` re-syncs at start, and without --no-dev it pulls the dev group: the deploy log shows it downloading mypy, ruff and pygments before doing anything. That is where the twelve minutes went, on a loop, and it still could not find alembic because the preDeployCommand ran before any of this had a chance to help. So every entrypoint now calls the venv binary directly and `uv run` is gone from infra/ entirely. apps/embedding already did exactly this, with a comment saying "avoid `uv run` reinstalling packages on restart" — the lesson was learned in one image and not the other four. Verified rather than assumed. My first diagnosis was that the workspace sync was dragging in torch from the two model sidecars; that was wrong — the old images contain no torch, because the sync was a no-op. The build log settled it: `Resolved 109 packages in 774ms`, `Checked in 1ms`, and the twelve minutes were in the deploy phase. What is now verified: the api image runs alembic and uvicorn with `--network none` in under a second, where the old one could not run at all without downloading first; all four images build; the local compose stack still comes up healthy with uvicorn as PID 1, and /readyz reports all three checks green. Also the first actual run of the combined agent-worker entrypoint, which had never been executed anywhere. Both children start and the worker retries its broker. On Railway it exited with "a child exited with 1; stopping the other" when RabbitMQ was unreachable — which is the designed behaviour working, not a bug: a dead worker takes the container down instead of leaving one that answers questions while every submitted repository sits in `queued` forever. --- infra/docker/agent.Dockerfile | 15 ++++++++---- infra/docker/api.Dockerfile | 23 ++++++++++++++++--- .../docker/railway-agent-worker-entrypoint.sh | 4 ++-- infra/docker/railway-agent-worker.Dockerfile | 4 +++- infra/docker/worker.Dockerfile | 13 +++++++++-- infra/railway/api.toml | 6 ++++- 6 files changed, 52 insertions(+), 13 deletions(-) diff --git a/infra/docker/agent.Dockerfile b/infra/docker/agent.Dockerfile index e498a0b..6271266 100644 --- a/infra/docker/agent.Dockerfile +++ b/infra/docker/agent.Dockerfile @@ -18,10 +18,17 @@ COPY apps/eval ./apps/eval COPY apps/embedding ./apps/embedding COPY apps/reranker ./apps/reranker -RUN uv sync --no-dev +# Scoped for the same reason as api.Dockerfile: the workspace contains two +# torch-dependent model sidecars this image does not run. +RUN uv sync --no-dev --package dcode-agent EXPOSE 8001 -CMD ["uv", "run", "--package", "dcode-agent", \ - "uvicorn", "dcode_agent.main:app", \ - "--host", "0.0.0.0", "--port", "8001"] +# The venv binary directly, not `uv run`. +# +# `uv run` re-syncs the environment at container start, and without --no-dev +# that means downloading mypy and ruff every single time. On the first Railway +# deploy it also could not find alembic at all, because the build-time sync had +# installed nothing (see the RUN above). apps/embedding already did it this way +# with a comment saying why; api, worker and agent had not caught up. +CMD ["sh", "-c", "exec /app/.venv/bin/uvicorn dcode_agent.main:app --host ${HOST:-::} --port ${PORT:-8001}"] diff --git a/infra/docker/api.Dockerfile b/infra/docker/api.Dockerfile index 4688875..6bd47c5 100644 --- a/infra/docker/api.Dockerfile +++ b/infra/docker/api.Dockerfile @@ -19,7 +19,18 @@ COPY apps/embedding ./apps/embedding COPY apps/reranker ./apps/reranker COPY infra/migrations ./infra/migrations -RUN uv sync --no-dev +# --package, not the whole workspace. +# +# `uv sync --no-dev` at the workspace root installs EVERY member, and two of +# them — dcode-embedding and dcode-reranker — depend on torch, transformers and +# sentence-transformers. This image runs none of that, and it was pulling +# several GB to sit unused. It was invisible locally because the layer was +# cached; the first clean build took twelve minutes. +# +# The sources are still copied so uv can resolve the workspace; only the +# installation is scoped. alembic is a dcode-api dependency, so the migration +# this image runs before each deploy still has it. +RUN uv sync --no-dev --package dcode-api EXPOSE 8000 @@ -28,5 +39,11 @@ EXPOSE 8000 # by nothing. The defaults are the previous literals, so compose is unchanged # — `sh -c exec` keeps uvicorn as the container's main process rather than # leaving a shell in front of it to swallow signals. -CMD ["sh", "-c", "exec uv run --package dcode-api uvicorn dcode_api.main:app \ - --host ${HOST:-0.0.0.0} --port ${PORT:-8000}"] +# The venv binary directly, not `uv run`. +# +# `uv run` re-syncs the environment at container start, and without --no-dev +# that means downloading mypy and ruff every single time. On the first Railway +# deploy it also could not find alembic at all, because the build-time sync had +# installed nothing (see the RUN above). apps/embedding already did it this way +# with a comment saying why; api, worker and agent had not caught up. +CMD ["sh", "-c", "exec /app/.venv/bin/uvicorn dcode_api.main:app --host ${HOST:-0.0.0.0} --port ${PORT:-8000}"] diff --git a/infra/docker/railway-agent-worker-entrypoint.sh b/infra/docker/railway-agent-worker-entrypoint.sh index d9c0f6b..84136ab 100755 --- a/infra/docker/railway-agent-worker-entrypoint.sh +++ b/infra/docker/railway-agent-worker-entrypoint.sh @@ -24,12 +24,12 @@ term() { trap term TERM INT echo "[entrypoint] starting agent on :${PORT:-8001}" -uv run --package dcode-agent uvicorn dcode_agent.main:app \ +/app/.venv/bin/uvicorn dcode_agent.main:app \ --host "${HOST:-::}" --port "${PORT:-8001}" & agent_pid=$! echo "[entrypoint] starting index worker" -uv run --package dcode-worker python -m dcode_worker.main & +/app/.venv/bin/python -m dcode_worker.main & worker_pid=$! # `wait -n` returns as soon as EITHER child exits. Waiting on both in sequence diff --git a/infra/docker/railway-agent-worker.Dockerfile b/infra/docker/railway-agent-worker.Dockerfile index 1a3b695..2d5d7aa 100644 --- a/infra/docker/railway-agent-worker.Dockerfile +++ b/infra/docker/railway-agent-worker.Dockerfile @@ -29,7 +29,9 @@ COPY apps/eval ./apps/eval COPY apps/embedding ./apps/embedding COPY apps/reranker ./apps/reranker -RUN uv sync --no-dev +# Both packages, and only those two. The workspace also contains the torch- +# dependent model sidecars, which this image does not run — see api.Dockerfile. +RUN uv sync --no-dev --package dcode-agent --package dcode-worker COPY infra/docker/railway-agent-worker-entrypoint.sh /usr/local/bin/dcode-entrypoint RUN chmod +x /usr/local/bin/dcode-entrypoint diff --git a/infra/docker/worker.Dockerfile b/infra/docker/worker.Dockerfile index ac944b4..000d050 100644 --- a/infra/docker/worker.Dockerfile +++ b/infra/docker/worker.Dockerfile @@ -18,6 +18,15 @@ COPY apps/eval ./apps/eval COPY apps/embedding ./apps/embedding COPY apps/reranker ./apps/reranker -RUN uv sync --no-dev +# Scoped for the same reason as api.Dockerfile: the workspace contains two +# torch-dependent model sidecars this image does not run. +RUN uv sync --no-dev --package dcode-worker -CMD ["uv", "run", "--package", "dcode-worker", "python", "-m", "dcode_worker.main"] +# The venv binary directly, not `uv run`. +# +# `uv run` re-syncs the environment at container start, and without --no-dev +# that means downloading mypy and ruff every single time. On the first Railway +# deploy it also could not find alembic at all, because the build-time sync had +# installed nothing (see the RUN above). apps/embedding already did it this way +# with a comment saying why; api, worker and agent had not caught up. +CMD ["/app/.venv/bin/python", "-m", "dcode_worker.main"] diff --git a/infra/railway/api.toml b/infra/railway/api.toml index 3e091b9..6a20ec3 100644 --- a/infra/railway/api.toml +++ b/infra/railway/api.toml @@ -31,7 +31,11 @@ watchPatterns = [ # *this* service (Deploy.md §6 PR 1). Running it anywhere else is how the column # ends up at a dimension nothing else agrees with. `upgrade head` is idempotent, # so running it on every deploy is safe. -preDeployCommand = ["uv run alembic -c infra/migrations/alembic.ini upgrade head"] +# The venv binary, not `uv run`. `uv run` re-syncs at start and, without +# --no-dev, downloads mypy and ruff before doing anything: the first deploy +# spent twelve minutes on that and then failed with "Failed to spawn: alembic", +# because the build-time sync had installed nothing at all. +preDeployCommand = ["/app/.venv/bin/alembic -c infra/migrations/alembic.ini upgrade head"] # The deep probe, not /healthz. /healthz answers ok with Postgres, Redis and # RabbitMQ all down; /readyz checks all three plus the embedding-dimension