feat: move to mq for markdown parsing

.env.example

@@ -0,0 +1,82 @@
+# Copy to .env and fill in. Defaults for everything here live in config.py;
+# .env only needs the values you actually want to override.
+#
+# The Makefile loads .env and supplies sensible local defaults for anything it
+# does not find, so a minimal .env is just OPENAI_API_KEY.
+
+# ---- Conversion ----
+OPENAI_API_KEY=
+OPENAI_MODEL=google/gemini-3.1-flash-lite-preview
+# LLM-assisted conversion (image descriptions). Slow and costs tokens per page.
+# Folded into the storage cache key, so changing it re-converts rather than
+# serving the previous setting's output.
+CONVERT_USE_LLM=false
+
+# ---- API ----
+ADMIN_API_KEY=dev-key
+
+# ---- Database (libSQL / Turso) ----
+# db.py picks the driver from this value:
+#   a path or file: URL  -> stdlib sqlite3   (local single-process, and CI)
+#   libsql:// or http:// -> libsql driver    (hosted Turso, or self-hosted sqld)
+#
+# The API and workers are separate processes and must share one database, so a
+# local SQLite file only works if you run a single process.
+TURSO_DATABASE_URL=http://localhost:8080
+# Hosted Turso only. Setting this forces the libsql driver regardless of URL.
+TURSO_AUTH_TOKEN=
+LIBSQL_PORT=8080
+
+# ---- Object storage ----
+# Any S3-compatible endpoint: minio locally, R2 or AWS in production.
+# Ports default off 9000/9001 since those commonly collide with other projects.
+S3_ENDPOINT=http://localhost:9100
+S3_BUCKET=markitdown
+S3_ACCESS_KEY_ID=minioadmin
+S3_SECRET_ACCESS_KEY=minioadmin
+S3_REGION=us-east-1
+MINIO_PORT=9100
+MINIO_CONSOLE_PORT=9101
+# Recorded on the document row. Actual expiry is a bucket lifecycle rule, so
+# every object for a document ages out together.
+DOC_TTL_DAYS=7
+
+# ---- Chunking ----
+# Smaller chunks lower peak memory per worker but add per-chunk parser startup.
+PDF_PAGES_PER_CHUNK=20
+# Below this page count a PDF is converted whole rather than split.
+PDF_MIN_PAGES_FOR_PARALLEL=40
+PAGE_SIZE=5000
+
+# ---- Queue (RabbitMQ) ----
+# Job distribution. The producer publishes one message per chunk; workers
+# consume with prefetch=1, so the broker hands each chunk to whoever is free.
+# Adding a worker pod adds throughput with no coordination.
+#
+# Port 5673 locally to avoid colliding with an existing 5672 broker.
+RABBITMQ_URL=amqp://guest:guest@localhost:5673/%2F
+RABBITMQ_QUEUE=markitdown.chunks
+RABBITMQ_EXCHANGE=markitdown
+RABBITMQ_PORT=5673
+RABBITMQ_MANAGEMENT_PORT=15673
+RABBITMQ_USER=guest
+RABBITMQ_PASSWORD=guest
+# Unacked messages a worker holds at once. 1 is what bounds memory to a chunk
+# rather than a document, and keeps dispatch fair.
+RABBITMQ_PREFETCH=1
+
+# Retry backoff tiers in seconds, one dedicated delay queue each; attempt N
+# waits tier N, then dead-letters back onto the work queue.
+#
+# Per-tier queues rather than a per-message TTL on one queue: RabbitMQ only
+# expires the message at the *head*, so a 300s message parked in front would
+# hold back a 5s message queued behind it.
+RETRY_DELAYS=5,30,300
+# Attempts per chunk. Exhausting them parks the message on
+# markitdown.chunks.failed and fails the whole job — a document silently
+# missing pages is worse than an error.
+MAX_ATTEMPTS=3
+
+# ---- Worker ----
+SOURCE_CACHE_DIR=/tmp/markitdown-src
+SOURCE_CACHE_MAX_BYTES=2147483648

.github/workflows/release.yaml

@@ -100,12 +100,17 @@ jobs:
           kubectl kustomize . | kubectl apply -f -
 
           # Update the deployment with the specific release tag
-          kubectl set image deployment/markitdown-server markitdown-server=ghcr.io/${{ github.repository_owner }}/markitdown-server:${VERSION} -n markitdown-server
+          # One image, two deployments: the producer (API) and the consumer
+          # (worker) must move together or a new worker could read job rows
+          # written by an old producer.
+          kubectl set image deployment/markitdown-api markitdown-api=ghcr.io/${{ github.repository_owner }}/markitdown-server:${VERSION} -n markitdown-server
+          kubectl set image deployment/markitdown-worker markitdown-worker=ghcr.io/${{ github.repository_owner }}/markitdown-server:${VERSION} -n markitdown-server
 
           # Wait for rollout to complete
-          kubectl rollout status deployment/markitdown-server -n markitdown-server --timeout=300s
+          kubectl rollout status deployment/markitdown-api -n markitdown-server --timeout=300s
+          kubectl rollout status deployment/markitdown-worker -n markitdown-server --timeout=300s
 
       - name: Verify deployment
         run: |
-          kubectl get pods -n markitdown-server -l app=markitdown-server
+          kubectl get pods -n markitdown-server
           kubectl get service -n markitdown-server markitdown-server

.github/workflows/test.yaml

@@ -29,12 +29,14 @@ jobs:
       - name: Install Python dependencies
         run: |
           pip install -r requirements.txt
-          pip install pytest reportlab
+          pip install pytest reportlab httpx "moto[s3]"
 
+      # No database or object-store service container needed: the tests run
+      # against stdlib sqlite3 and an in-process moto S3. db.py picks the
+      # sqlite driver for any non-server DATABASE_URL, so the same code paths
+      # are exercised here as in the cluster.
       - name: Run unit tests
-        env:
-          PYTHONPATH: ${{ github.workspace }}
-        run: pytest tests/ -v
+        run: pytest -v
 
   k8s-benchmark:
     name: PDF Benchmark (Kubernetes)
@@ -77,12 +79,21 @@ jobs:
       - name: Create secret
         run: |
           kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f -
-          # PDF conversion never calls the LLM (that path is images only), so a
-          # placeholder key is enough to satisfy the import-time lookup.
+          # Conversion only calls the LLM when use_llm is requested, which the
+          # benchmark does not do, so a placeholder key is enough.
+          # RABBITMQ_URL points at the in-cluster broker from rabbitmq.yaml:
+          # the runner has no route to the shared linda-namespace broker the
+          # real secret uses. RABBITMQ_USER/PASSWORD are what that broker boots
+          # with, so the two must agree.
           kubectl create secret generic markitdown-server-secret \
             -n "$NAMESPACE" \
             --from-literal=OPENAI_API_KEY=ci-placeholder \
             --from-literal=ADMIN_API_KEY=ci-test-key \
+            --from-literal=S3_ACCESS_KEY_ID=ci-minio-user \
+            --from-literal=S3_SECRET_ACCESS_KEY=ci-minio-password \
+            --from-literal=RABBITMQ_USER=ci-rabbit-user \
+            --from-literal=RABBITMQ_PASSWORD=ci-rabbit-password \
+            --from-literal=RABBITMQ_URL='amqp://ci-rabbit-user:ci-rabbit-password@markitdown-rabbitmq:5672/%2F' \
             --dry-run=client -o yaml | kubectl apply -f -
 
       - name: Deploy
@@ -91,18 +102,24 @@ jobs:
           kustomize edit set image \
             "ghcr.io/sirily11/markitdown-server=markitdown-server:${IMAGE_TAG}"
           kubectl apply -k .
-          # Drop the HPA for the benchmark. Conversion pegs the CPU, so the HPA
-          # scales to maxReplicas mid-run and then holds there for its 300s
-          # scale-down window. Two replicas behind the service make per-book
-          # timing and peak-memory attribution meaningless, since requests and
-          # the cgroup reads can land on different pods.
-          kubectl delete hpa markitdown-server-hpa -n "$NAMESPACE" --ignore-not-found
-          kubectl scale deployment/markitdown-server -n "$NAMESPACE" --replicas=1
+          # Drop the worker HPA and pin the worker to one replica. Conversion
+          # pegs the CPU, so the HPA would scale up mid-run and hold there for
+          # its scale-down window; with a worker fleet, peak memory is spread
+          # across pods and no single cgroup reading means anything.
+          kubectl delete hpa markitdown-worker-hpa -n "$NAMESPACE" --ignore-not-found
+          kubectl delete hpa markitdown-api-hpa -n "$NAMESPACE" --ignore-not-found
+          kubectl scale deployment/markitdown-worker -n "$NAMESPACE" --replicas=1
 
       - name: Wait for rollout
         run: |
-          kubectl rollout status deployment/markitdown-redis -n "$NAMESPACE" --timeout=300s || true
-          kubectl rollout status deployment/markitdown-server -n "$NAMESPACE" --timeout=300s
+          # State first: the API and worker both fail readiness until these are
+          # up, so rolling them out in order keeps the failure legible.
+          kubectl rollout status deployment/markitdown-libsql -n "$NAMESPACE" --timeout=300s
+          kubectl rollout status deployment/markitdown-minio -n "$NAMESPACE" --timeout=300s
+          kubectl wait --for=condition=complete job/markitdown-minio-setup \
+            -n "$NAMESPACE" --timeout=300s
+          kubectl rollout status deployment/markitdown-api -n "$NAMESPACE" --timeout=300s
+          kubectl rollout status deployment/markitdown-worker -n "$NAMESPACE" --timeout=300s
 
       # No port-forward here on purpose: the benchmark restarts the pod between
       # books, which tears a forward down, so the script manages its own.
@@ -167,10 +184,16 @@ jobs:
           # Per-book logs captured during the run. The pod is restarted between
           # books, so these are the only copy of an earlier book's output.
           cp -r /tmp/benchmark-pod-logs /tmp/k8s-logs/per-book 2>/dev/null || true
-          kubectl logs -n "$NAMESPACE" -l app=markitdown-server --tail=-1 \
-            > /tmp/k8s-logs/server-final.log 2>&1 || true
+          kubectl logs -n "$NAMESPACE" -l app=markitdown-api --tail=-1 \
+            > /tmp/k8s-logs/api-final.log 2>&1 || true
+          kubectl logs -n "$NAMESPACE" -l app=markitdown-worker --tail=-1 \
+            > /tmp/k8s-logs/worker-final.log 2>&1 || true
+          kubectl logs -n "$NAMESPACE" -l app=markitdown-libsql --tail=200 \
+            > /tmp/k8s-logs/libsql.log 2>&1 || true
+          kubectl logs -n "$NAMESPACE" -l app=markitdown-minio --tail=200 \
+            > /tmp/k8s-logs/minio.log 2>&1 || true
           kubectl get pods -n "$NAMESPACE" -o wide > /tmp/k8s-logs/pods.txt 2>&1 || true
-          kubectl describe pods -n "$NAMESPACE" -l app=markitdown-server \
+          kubectl describe pods -n "$NAMESPACE" \
             > /tmp/k8s-logs/describe.txt 2>&1 || true
 
       - name: Upload diagnostics

.gitignore

@@ -3,4 +3,14 @@
 __pycache__
 .DS_Store
 .vercel
-secrets.yaml
+# Real credentials. The trailing * also covers editor/backup spill
+# (secrets.yaml.bak, .orig, .save); secrets.example.yaml is not matched and
+# stays tracked as the template.
+secrets.yaml*
+.venv
+venv
+.pytest_cache
+# Local sqlite database, used when TURSO_DATABASE_URL points at a file.
+*.db
+*.db-wal
+*.db-shm

Dockerfile

@@ -28,5 +28,6 @@ EXPOSE 8000
 ENV PYTHONPATH=/app
 ENV PYTHONUNBUFFERED=1
 
-# Run the server
+# One image, two roles. The default is the API (producer); worker pods override
+# the command with ["python", "-m", "worker"] to run the consumer instead.
 CMD ["uvicorn", "api.index:app", "--host", "0.0.0.0", "--port", "8000"]

Makefile

@@ -1,3 +1,37 @@
+# Local development.
+#
+#   make up      rabbitmq + minio
+#   make api     the producer (uvicorn, with reload)
+#   make worker  a consumer — run several to add capacity
+#   make test    the test suite
+#
+# The database is remote (Turso); only the broker and object storage run here.
+
+ifneq (,$(wildcard .env))
+-include .env
+export
+endif
+
+PY   := $(if $(wildcard .venv/bin/python),.venv/bin/python,python3)
+PORT ?= 8000
+
+.PHONY: up down api worker test docker
+
+up:
+	docker compose up -d
+
+down:
+	docker compose down
+
+api:
+	$(PY) -m uvicorn api.index:app --reload --host 0.0.0.0 --port $(PORT)
+
+worker:
+	$(PY) -m worker
+
+test:
+	$(PY) -m pytest -q
+
 docker:
 	docker buildx create --use --name multi-arch-builder || true
 	docker buildx build \
@@ -8,4 +42,4 @@ docker:
 		--build-arg BUILD_TIME=$$(date -u +'%Y-%m-%d_%H:%M:%S') \
 		--build-arg COMMIT_HASH=$$(git rev-parse --short HEAD) \
 		--push \
-		.
+		.