ci: parallelise jobs, remove HF download, commit source_list fixture

.github/workflows/ci.yaml

@@ -15,8 +15,8 @@ defaults:
     shell: bash
 
 jobs:
-  build-backend-docker:
-    runs-on: self-hosted
+  lint-backend:
+    runs-on: ubuntu-latest
     steps:
     - name: Setup python
       uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
@@ -29,22 +29,82 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4.3.1
 
-    - name: Setup prereqs
-      run: |
-        make init-dev
+    - name: Install deps
+      run: cd backend && make init-dev
 
-    - name: Run formatting checks
-      run: |
-        make check
+    - name: Lint
+      run: cd backend && make check
+
+  lint-frontend:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Setup python
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+      with:
+        python-version: '3.12'
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e  # v6.8.0
+
+    - name: Checkout code
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4.3.1
+
+    - name: Install deps
+      run: cd frontend && make init-dev
+
+    - name: Lint
+      run: cd frontend && make check
+
+  lint-evaluation:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Setup python
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+      with:
+        python-version: '3.12'
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e  # v6.8.0
+
+    - name: Checkout code
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4.3.1
+
+    - name: Install deps
+      run: cd evaluation && make init-dev
+
+    - name: Lint
+      run: cd evaluation && make check
+
+  test:
+    needs: [lint-backend]
+    runs-on: self-hosted
+    steps:
+    - name: Setup python
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5.6.0
+      with:
+        python-version: '3.12'
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e  # v6.8.0
+
+    - name: Checkout code
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4.3.1
+
+    - name: Install deps
+      run: cd backend && make init-dev
 
     - name: Run unit tests
       working-directory: backend
       run: |
-        uv pip install huggingface_hub[cli]
-        huggingface-cli download --repo-type dataset The-OpenROAD-Project/ORAssistant_RAG_Dataset --include source_list.json --local-dir data/
         cp .env.test .env
         make test
 
+  docker-build:
+    needs: [test, lint-frontend, lint-evaluation]
+    runs-on: self-hosted
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5  # v4.3.1
+
     - name: Build Docker images
-      run: |
-        docker compose build
+      run: docker compose build --parallel

.gitignore

@@ -1,7 +1,8 @@
 .env
 *.ipynb
 __pycache__/
-backend/data/
+backend/data/*
+!backend/data/source_list.json
 backend/src/*.json
 *.pyc
 *.egg-info/

Makefile

@@ -26,6 +26,12 @@ check:
 	@. ./backend/.venv/bin/activate && \
 		pre-commit run --all-files
 
+.PHONY: check-ci
+check-ci:
+	@for folder in $(FOLDERS); do \
+	   (cd $$folder && make check && cd ../) || exit 1; \
+		done
+
 .PHONY: docker-up
 docker-up:
 	@docker compose -f docker-compose.yml up --build --wait