From 24e044b638ff3cdbc84306e1e09539aaf3691f4e Mon Sep 17 00:00:00 2001 From: Yash-Chindam Date: Sun, 30 Aug 2026 14:43:20 +0530 Subject: [PATCH 1/4] chore: add locked Playwright test toolchain --- package-lock.json | 78 ++++++++++++++++++++++++++++++++++++++++++++ package.json | 11 +++++++ playwright.config.ts | 29 ++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 playwright.config.ts diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..d7c3b74 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,78 @@ +{ + "name": "local-llm-router-e2e", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "local-llm-router-e2e", + "version": "0.1.0", + "devDependencies": { + "@playwright/test": "^1.55.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz", + "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..8d30744 --- /dev/null +++ b/package.json @@ -0,0 +1,11 @@ +{ + "name": "local-llm-router-e2e", + "private": true, + "version": "0.1.0", + "scripts": { + "test:e2e": "playwright test" + }, + "devDependencies": { + "@playwright/test": "^1.55.0" + } +} diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..e3457cc --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,29 @@ +import { defineConfig } from "@playwright/test"; + +const port = process.env.TEST_PORT ?? "8000"; +const baseURL = `http://127.0.0.1:${port}`; + +export default defineConfig({ + testDir: "./tests/e2e", + fullyParallel: true, + forbidOnly: Boolean(process.env.CI), + retries: process.env.CI ? 2 : 0, + workers: process.env.CI ? 1 : undefined, + reporter: process.env.CI ? [["github"], ["html", { open: "never" }]] : "list", + use: { + baseURL, + extraHTTPHeaders: { Authorization: "Bearer e2e-key" }, + trace: "retain-on-failure", + }, + webServer: { + command: `python -m uvicorn llm_router.app:app --app-dir src --host 127.0.0.1 --port ${port}`, + url: `${baseURL}/healthz`, + reuseExistingServer: !process.env.CI, + timeout: 30_000, + env: { + ...process.env, + ROUTER_API_KEYS: "e2e-key", + ROUTER_ENVIRONMENT: "test", + }, + }, +}); From 217e1b5819b9dc1b5859d44aefb151cbd4a327f1 Mon Sep 17 00:00:00 2001 From: Yash-Chindam Date: Sun, 30 Aug 2026 14:43:20 +0530 Subject: [PATCH 2/4] test: cover gateway behavior with Playwright --- tests/e2e/api.spec.ts | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/e2e/api.spec.ts diff --git a/tests/e2e/api.spec.ts b/tests/e2e/api.spec.ts new file mode 100644 index 0000000..f4f415f --- /dev/null +++ b/tests/e2e/api.spec.ts @@ -0,0 +1,43 @@ +import { expect, test } from "@playwright/test"; + +test("serves health and authenticated model catalog", async ({ request }) => { + const health = await request.get("/healthz"); + expect(health.ok()).toBeTruthy(); + await expect(health.json()).resolves.toEqual({ status: "healthy" }); + + const models = await request.get("/v1/models"); + expect(models.ok()).toBeTruthy(); + expect((await models.json()).data).toEqual( + expect.arrayContaining([expect.objectContaining({ id: "general-local" })]), + ); +}); + +test("routes an OpenAI-compatible extraction request end to end", async ({ request }) => { + const response = await request.post("/v1/chat/completions", { + data: { + model: "auto", + messages: [{ role: "user", content: "Extract the account fields as JSON" }], + routing: { privacy: "restricted", latency_tier: "interactive" }, + }, + }); + expect(response.status()).toBe(200); + expect(response.headers()["x-route-model"]).toBe("small-specialist"); + const body = await response.json(); + expect(body).toMatchObject({ + object: "chat.completion", + model: "small-specialist", + routing: { task: "extraction" }, + }); + expect(body.routing.model_revision).toBeTruthy(); + expect(body.choices[0].message.role).toBe("assistant"); +}); + +test("rejects invalid credentials", async ({ playwright }, testInfo) => { + const anonymous = await playwright.request.newContext({ + baseURL: testInfo.project.use.baseURL, + extraHTTPHeaders: { Authorization: "Bearer invalid-key" }, + }); + const response = await anonymous.get("/v1/models"); + expect(response.status()).toBe(401); + await anonymous.dispose(); +}); From cab44c4375df10d60b1ee163e289e69a57027615 Mon Sep 17 00:00:00 2001 From: Yash-Chindam Date: Sun, 30 Aug 2026 14:43:38 +0530 Subject: [PATCH 3/4] ci: require Playwright before release image build --- .github/workflows/ci.yml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6fb6f6f..e4e29d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,10 +44,34 @@ jobs: - run: python -m pip install -e ".[dev]" - run: pytest tests/integration + e2e: + name: Playwright E2E + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: python -m pip install -e ".[dev]" + - run: npm ci + - run: npx playwright install --with-deps chromium + - run: npm run test:e2e + - if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: playwright-report + path: playwright-report/ + retention-days: 14 + container: name: Release image validation runs-on: ubuntu-latest - needs: [unit, integration] + needs: [unit, integration, e2e] steps: - uses: actions/checkout@v4 - uses: docker/setup-buildx-action@v3 From 994bc63a34eaca70fd0dc3577d92eaffe8e2824f Mon Sep 17 00:00:00 2001 From: Yash-Chindam Date: Sun, 30 Aug 2026 14:44:07 +0530 Subject: [PATCH 4/4] docs: document Playwright end-to-end workflow --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 54e0f8b..7bbe0b2 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,12 @@ The complete architecture and design targets are documented in ## Development -Requires Python 3.11 or newer. +Requires Python 3.11 or newer and Node.js 20 or newer. ```bash python -m venv .venv python -m pip install -e ".[dev]" +npm ci python -m uvicorn llm_router.app:app --app-dir src --reload ``` @@ -43,9 +44,14 @@ ruff format --check . ruff check . mypy pytest tests/unit tests/integration --cov=llm_router --cov-report=term-missing +npx playwright install chromium +npm run test:e2e docker build -t local-llm-router:dev . ``` +CI reports unit/static analysis, integration, and Playwright end-to-end tests separately. +The release-image build starts only after all three test layers pass. + ## Runtime settings All settings use the `ROUTER_` prefix.