ci: parallelise secret CI, cache HF dataset, speed up docker build

.github/workflows/ci-secret.yaml

@@ -11,13 +11,63 @@ on:
       - 'evaluation/**'
       - 'Makefile'
       - 'docker-compose.yml'
+      - 'docker-compose.ci.yml'
 
 defaults:
   run:
     shell: bash
 
 jobs:
-  build-backend-docker:
+  lint-backend:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Setup python
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+      with:
+        python-version: '3.12'
+    - name: Install uv
+      uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e  # v6.8.0
+    - name: Checkout code
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4.3.1
+    - name: Install deps
+      run: cd backend && make init-dev
+    - name: Lint
+      run: cd backend && make check
+
+  lint-frontend:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Setup python
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+      with:
+        python-version: '3.12'
+    - name: Install uv
+      uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e  # v6.8.0
+    - name: Checkout code
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4.3.1
+    - name: Install deps
+      run: cd frontend && make init-dev
+    - name: Lint
+      run: cd frontend && make check
+
+  lint-evaluation:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Setup python
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+      with:
+        python-version: '3.12'
+    - name: Install uv
+      uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e  # v6.8.0
+    - name: Checkout code
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4.3.1
+    - name: Install deps
+      run: cd evaluation && make init-dev
+    - name: Lint
+      run: cd evaluation && make check
+
+  test:
+    needs: [lint-backend]
     runs-on: self-hosted
     steps:
     - name: Setup python
@@ -28,19 +78,32 @@ jobs:
       uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e  # v6.8.0
     - name: Checkout code
       uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4.3.1
-    - name: Setup prereqs
-      run: |
-        make init-dev
-    - name: Run formatting checks
-      run: |
-        make check
+    - name: Install deps
+      run: cd backend && make init-dev
     - name: Run unit tests
       working-directory: backend
       run: |
-        uv pip install huggingface_hub[cli]
-        huggingface-cli download --repo-type dataset The-OpenROAD-Project/ORAssistant_RAG_Dataset --include source_list.json --local-dir data/
         cp .env.test .env
         make test
+
+  docker-eval:
+    needs: [test, lint-frontend, lint-evaluation]
+    runs-on: self-hosted
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4.3.1
+
+    - name: Download HF dataset
+      run: |
+        if [ ! -d "data" ] || [ -z "$(ls -A data 2>/dev/null)" ]; then
+          pip install huggingface_hub
+          huggingface-cli download The-OpenROAD-Project/ORAssistant_RAG_Dataset \
+            --local-dir ./data
+          echo "Dataset downloaded"
+        else
+          echo "Dataset already cached on runner, skipping download"
+        fi
+
     - name: Populate environment variables
       run: |
         cp backend/.env.example backend/.env
@@ -54,14 +117,16 @@ jobs:
         sed -i 's|{{GOOGLE_PROJECT_ID}}|${{ secrets.GOOGLE_PROJECT_ID }}|g' evaluation/.env
         sed -i 's|{{PATH_TO_GOOGLE_APPLICATION_CREDENTIALS}}|src/secret.json|g' evaluation/.env
         sed -i 's|HF_TOKEN=|HF_TOKEN=${{ secrets.HF_TOKEN }}|g' evaluation/.env
+
     - name: Copy Google credentials
       env:
         GOOGLE_SECRET_JSON: ${{ secrets.PATH_TO_GOOGLE_APPLICATION_CREDENTIALS }}
       run: |
         make seed-credentials
-    - name: Build Docker image
+
+    - name: Build and start Docker stack
       run: |
-        make docker-up
+        make docker-up-ci
 
     - name: Run LLM CI
       id: llm_tests
@@ -85,7 +150,6 @@ jobs:
         echo "=== COMPLETE POSTGRES LOGS ==="
         docker logs postgres 2>&1 || echo "Failed to get postgres logs"
 
-        # Save to files for artifacts
         docker logs backend > docker-logs/backend.log 2>&1 || echo "Failed to capture backend logs"
         docker logs frontend > docker-logs/frontend.log 2>&1 || echo "Failed to capture frontend logs"
         docker logs postgres > docker-logs/postgres.log 2>&1 || echo "Failed to capture postgres logs"
@@ -116,4 +180,4 @@ jobs:
     - name: Teardown
       if: always()
       run: |
-        make docker-down
+        make docker-down-ci

Makefile

@@ -36,10 +36,18 @@ check-ci:
 docker-up:
 	@docker compose -f docker-compose.yml up --build --wait
 
+.PHONY: docker-up-ci
+docker-up-ci:
+	@docker compose -f docker-compose.yml -f docker-compose.ci.yml up --build --wait
+
 .PHONY: docker-down
 docker-down:
 	@docker compose -f docker-compose.yml down --volumes --remove-orphans
 
+.PHONY: docker-down-ci
+docker-down-ci:
+	@docker compose -f docker-compose.yml -f docker-compose.ci.yml down --volumes --remove-orphans
+
 .PHONY: docker-dev
 docker-dev:
 	@docker compose -f docker-compose.yml -f docker-compose.dev.yml up --build --wait

backend/Dockerfile

@@ -20,12 +20,15 @@ RUN pip install uv
 COPY ./pyproject.toml /ORAssistant-backend/pyproject.toml
 COPY . .
 
-RUN uv venv .venv && uv sync --dev && uv run /ORAssistant-backend/src/post_install.py
+RUN uv venv .venv && uv sync && uv run /ORAssistant-backend/src/post_install.py
 
-RUN git clone https://huggingface.co/datasets/The-OpenROAD-Project/ORAssistant_RAG_Dataset && \
+ARG SKIP_HF_DOWNLOAD=false
+RUN if [ "$SKIP_HF_DOWNLOAD" = "false" ]; then \
+    git clone https://huggingface.co/datasets/The-OpenROAD-Project/ORAssistant_RAG_Dataset && \
     mkdir -p data && \
     mv ORAssistant_RAG_Dataset/* data/ && \
-    rm -rf ORAssistant_RAG_Dataset
+    rm -rf ORAssistant_RAG_Dataset; \
+fi
 
 EXPOSE 8000
 

backend/Makefile

@@ -28,7 +28,7 @@ build-docs:
 
 .PHONY: test
 test:
-	@uv run pytest
+	@uv run pytest -n auto
 
 .PHONY: mcp
 mcp:

docker-compose.ci.yml

@@ -0,0 +1,17 @@
+services:
+  backend:
+    build:
+      context: ./backend
+      args:
+        SKIP_HF_DOWNLOAD: "true"
+    volumes:
+      - faiss_data:/ORAssistant-backend/faiss_db
+      - ./data:/ORAssistant-backend/data
+    environment:
+      HEALTHCHECK_START_PERIOD: "300s"
+      HEALTHCHECK_INTERVAL: "10s"
+      HEALTHCHECK_RETRIES: "30"
+
+volumes:
+  faiss_data:
+    driver: local