From c344b8d14d6244a7179dffee6efd41ab2c9fb84f Mon Sep 17 00:00:00 2001 From: Bart Date: Wed, 28 Oct 2026 17:26:07 +0100 Subject: [PATCH 01/13] feat(evals): run the MCP host evals through OpenRouter and in CI Route the eval dataset through OpenRouter (same google/gemini-2.5-flash model) instead of the Google provider, add EVAL_MODEL to swap the model, and declare ai and @openrouter/ai-sdk-provider as dev dependencies. Add a weekly and on-demand MCP evals workflow modeled on Azure smoke. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/mcp-evals.yml | 62 ++++++++++++++ README.md | 5 +- ...-comprehensive.json => comprehensive.json} | 84 +++++++++++++++---- package.json | 4 +- .../{evals-gemini.spec.mjs => evals.spec.mjs} | 13 ++- yarn.lock | 41 +++++++++ 6 files changed, 185 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/mcp-evals.yml rename data/evals/{gemini-comprehensive.json => comprehensive.json} (87%) rename tests/mcp/{evals-gemini.spec.mjs => evals.spec.mjs} (71%) diff --git a/.github/workflows/mcp-evals.yml b/.github/workflows/mcp-evals.yml new file mode 100644 index 0000000..ca27f86 --- /dev/null +++ b/.github/workflows/mcp-evals.yml @@ -0,0 +1,62 @@ +name: MCP evals + +# The LLM-driven eval dataset through OpenRouter, weekly and on demand. +on: + workflow_dispatch: + inputs: + model: + description: "OpenRouter model ID for every case (empty keeps the dataset's model)" + required: false + default: "" + schedule: + - cron: "0 7 * * 1" + +permissions: + contents: read + +jobs: + mcp-evals: + runs-on: ubuntu-latest + # The spec itself times out after 90 minutes. + timeout-minutes: 100 + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + EVAL_MODEL: ${{ inputs.model }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Use Node.js 22 + uses: actions/setup-node@v4 + with: + node-version: 22.x + + - name: Install dependencies + run: yarn + + # The deployer case runs tflocal; the wrapper would garble its output. + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 + with: + terraform_wrapper: false + + - name: Install tflocal and the Snowflake CLI + run: pipx install terraform-local && pipx install snowflake-cli + + # Pulled up front so a start inside a case doesn't time out on the download. + - name: Pull the LocalStack images + run: | + docker pull localstack/localstack-pro:latest + docker pull localstack/snowflake:latest + + - name: Run the eval dataset + run: yarn test:mcp:evals + + - name: Keep the eval results + if: always() + uses: actions/upload-artifact@v4 + with: + name: mcp-evals-results + path: .mcp-test-results + if-no-files-found: ignore diff --git a/README.md b/README.md index 4ab0b40..40bd1ff 100644 --- a/README.md +++ b/README.md @@ -262,12 +262,13 @@ This repository includes [MCP Server Tester](https://github.com/gleanwork/mcp-se ```bash yarn test:mcp:direct ``` -- Run Gemini-based MCP host evals: +- Run MCP host evals through [OpenRouter](https://openrouter.ai) (default model `google/gemini-2.5-flash`; set `EVAL_MODEL` to any OpenRouter model ID to override it): ```bash - export GOOGLE_GENERATIVE_AI_API_KEY="" + export OPENROUTER_API_KEY="" export LOCALSTACK_AUTH_TOKEN="" yarn test:mcp:evals ``` + The [MCP evals workflow](./.github/workflows/mcp-evals.yml) runs them weekly and on demand; it needs the `OPENROUTER_API_KEY` repository secret. - Open the latest MCP Server Tester HTML report: ```bash npx mcp-server-tester open diff --git a/data/evals/gemini-comprehensive.json b/data/evals/comprehensive.json similarity index 87% rename from data/evals/gemini-comprehensive.json rename to data/evals/comprehensive.json index a03bc9c..5225343 100644 --- a/data/evals/gemini-comprehensive.json +++ b/data/evals/comprehensive.json @@ -1,13 +1,17 @@ { - "name": "localstack-mcp-gemini-comprehensive", - "description": "Scenario-based Gemini evals covering LocalStack MCP tools.", + "name": "localstack-mcp-comprehensive", + "description": "Scenario-based OpenRouter evals covering LocalStack MCP tools.", "cases": [ { "id": "scenario-management-status", "description": "localstack-management: status check of the AWS emulator.", "mode": "mcp_host", "scenario": "Check the current status of the LocalStack AWS emulator using the LocalStack management tool. If it is not running, start it. Report whether it is ready to accept requests.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -28,7 +32,11 @@ "description": "localstack-deployer: deploy the sample Terraform project.", "mode": "mcp_host", "scenario": "Deploy the Terraform project located in the data/sample-terraform directory of this repository to LocalStack using the LocalStack deployer tool, then report the Terraform outputs.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -45,7 +53,11 @@ "description": "localstack-aws-client: run a pinned AWS CLI command.", "mode": "mcp_host", "scenario": "Using the LocalStack AWS client tool, run the AWS CLI command \"s3 ls\" against LocalStack. Pass the command string exactly as s3 ls, with no aws or awslocal prefix and no extra flags. Then tell me which S3 buckets exist.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -66,7 +78,11 @@ "description": "localstack-logs-analysis: errors analysis of container logs.", "mode": "mcp_host", "scenario": "Use the LocalStack logs analysis tool to run an 'errors' analysis of the recent LocalStack container logs and report which AWS API calls failed.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -87,7 +103,11 @@ "description": "localstack-app-inspector: status check plus trace listing.", "mode": "mcp_host", "scenario": "Using the LocalStack App Inspector tool, first check whether App Inspector is currently enabled, enable it only if it is disabled, and then list the most recent App Inspector traces.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -113,7 +133,11 @@ "description": "localstack-iam-policy-analyzer: read mode, then set SOFT_MODE.", "mode": "mcp_host", "scenario": "Check the current IAM enforcement mode in LocalStack using the IAM policy analyzer, then set the enforcement mode to SOFT_MODE with the set-mode action. Perform the set-mode call even if the current mode is already SOFT_MODE, so the setting is explicitly confirmed.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -139,7 +163,11 @@ "description": "localstack-state-management: read-only inspect of runtime state.", "mode": "mcp_host", "scenario": "Using the LocalStack state management tool, inspect the in-memory state of the running LocalStack instance and tell me which AWS services currently hold state. Do not export, import, or reset anything.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -161,8 +189,8 @@ "mode": "mcp_host", "scenario": "Using the LocalStack Cloud Pods tool, save the current LocalStack state to a remote Cloud Pod named mcptester-e2e-pod. If saving fails because a pod with that name already exists, delete the existing pod and save again. If a save, load, or delete attempt times out, retry that same operation once before giving up. After saving, load the Cloud Pod mcptester-e2e-pod back into LocalStack. Finally, delete the Cloud Pod mcptester-e2e-pod so nothing is left behind.", "mcpHostConfig": { - "provider": "google", - "model": "gemini-2.5-flash", + "provider": "openrouter", + "model": "google/gemini-2.5-flash", "temperature": 0, "maxToolCalls": 15 }, @@ -196,7 +224,11 @@ "description": "localstack-chaos-injector: inject, verify, and clear latency.", "mode": "mcp_host", "scenario": "Use the LocalStack chaos injector to add 100 milliseconds of network latency to LocalStack, then read back the currently configured latency to verify it was applied, and finally clear the latency so LocalStack returns to normal.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -227,7 +259,11 @@ "description": "localstack-extensions: browse the marketplace catalog.", "mode": "mcp_host", "scenario": "Browse the LocalStack Extensions marketplace and show me which extensions are available to install. I want the marketplace catalog of available extensions, not the list of extensions that are already installed.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -248,7 +284,11 @@ "description": "localstack-aws-replicator: read-only supported-resources and job listing.", "mode": "mcp_host", "scenario": "Using the LocalStack AWS replicator tool, first show me which resource types the AWS replicator supports for replication into LocalStack, and then list any replication jobs that currently exist. Do not start any replication job.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -274,7 +314,11 @@ "description": "localstack-ephemeral-instances: read-only listing.", "mode": "mcp_host", "scenario": "List all LocalStack Ephemeral Instances that currently exist in my LocalStack Cloud account using the ephemeral instances tool, and summarize their names and statuses. Do not create, modify, or delete any instance.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -295,7 +339,11 @@ "description": "localstack-docs: documentation search.", "mode": "mcp_host", "scenario": "Search the official LocalStack documentation for how IAM policy enforcement modes work in LocalStack and summarize what the docs say. Use the LocalStack documentation search tool, not the IAM tools that act on the running instance.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, "iterations": 5, "accuracyThreshold": 0.8, "expect": { @@ -311,8 +359,8 @@ "mode": "mcp_host", "scenario": "Work with the LocalStack Snowflake emulator: 1) Using the LocalStack management tool with service set to snowflake, start the Snowflake emulator. If it reports that it is already running, continue to step 2. If starting fails because a LocalStack container is already running, that container is the AWS emulator: stop it with the management tool, then start the Snowflake emulator with service snowflake again. If a start attempt times out, keep checking the Snowflake status with the management tool (service snowflake) until it reports ready β€” do not give up on a timeout. 2) Use the LocalStack Snowflake client to check the connection to the emulator. 3) Use the LocalStack Snowflake client to execute the SQL file data/sample-sql/snowflake_test.sql. 4) Run the query SHOW TABLES IN demo_db.public and tell me whether the customers table exists.", "mcpHostConfig": { - "provider": "google", - "model": "gemini-2.5-flash", + "provider": "openrouter", + "model": "google/gemini-2.5-flash", "temperature": 0, "maxToolCalls": 15 }, diff --git a/package.json b/package.json index 8c3b7ad..25bbd1d 100644 --- a/package.json +++ b/package.json @@ -15,7 +15,7 @@ "format": "prettier --write .", "test": "jest", "test:mcp:direct": "yarn build && playwright test -c playwright.config.mjs tests/mcp/direct.spec.mjs tests/mcp/azure-offline.spec.mjs", - "test:mcp:evals": "yarn build && playwright test -c playwright.config.mjs tests/mcp/evals-gemini.spec.mjs", + "test:mcp:evals": "yarn build && playwright test -c playwright.config.mjs tests/mcp/evals.spec.mjs", "test:mcp": "yarn test:mcp:direct && yarn test:mcp:evals" }, "dependencies": { @@ -26,10 +26,12 @@ "devDependencies": { "@clack/prompts": "^1.8.1", "@gleanwork/mcp-server-tester": "1.1.1", + "@openrouter/ai-sdk-provider": "^2.0.0", "@playwright/test": "^1.64.1", "@types/dockerode": "^4.0.2", "@types/jest": "^30.0.0", "@xmcp-dev/compiler": "^1.9.0", + "ai": "^6.0.0", "audit-ci": "^7.1.0", "esbuild": "^0.28.2", "eslint-config-prettier": "^10.1.8", diff --git a/tests/mcp/evals-gemini.spec.mjs b/tests/mcp/evals.spec.mjs similarity index 71% rename from tests/mcp/evals-gemini.spec.mjs rename to tests/mcp/evals.spec.mjs index 528c337..6a368dc 100644 --- a/tests/mcp/evals-gemini.spec.mjs +++ b/tests/mcp/evals.spec.mjs @@ -9,14 +9,21 @@ function requireEnv(name) { return value; } -test.describe("Gemini comprehensive eval", () => { +test.describe("OpenRouter comprehensive eval", () => { test.describe.configure({ timeout: 5400000 }); test("single comprehensive eval dataset passes", async ({ mcp }, testInfo) => { - requireEnv("GOOGLE_GENERATIVE_AI_API_KEY"); + requireEnv("OPENROUTER_API_KEY"); requireEnv("LOCALSTACK_AUTH_TOKEN"); - const dataset = await loadEvalDataset("./data/evals/gemini-comprehensive.json"); + const dataset = await loadEvalDataset("./data/evals/comprehensive.json"); + // EVAL_MODEL swaps the OpenRouter model for every case, e.g. anthropic/claude-sonnet-4.5. + const model = process.env.EVAL_MODEL?.trim(); + if (model) { + for (const evalCase of dataset.cases) { + if (evalCase.mcpHostConfig) evalCase.mcpHostConfig.model = model; + } + } const result = await runEvalDataset({ dataset }, { mcp, testInfo }); const caseResults = result.caseResults || []; const passed = caseResults.filter((entry) => entry?.pass === true).length; diff --git a/yarn.lock b/yarn.lock index 443a457..ac2aaa5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -37,6 +37,15 @@ "@ai-sdk/provider-utils" "4.0.40" "@vercel/oidc" "3.2.0" +"@ai-sdk/gateway@3.0.238": + version "3.0.238" + resolved "https://registry.yarnpkg.com/@ai-sdk/gateway/-/gateway-3.0.238.tgz#b1465e928ad7b24925afbcdeaf09197b64e1c8a7" + integrity sha512-Kqdof17ZXdaKaHsoY/Z4HSbji86DVbIaG0kSgQwrKlEQIfp9Aip2S6xbba063EtLKWx0m/psvzCBYP73YUyWKA== + dependencies: + "@ai-sdk/provider" "3.0.19" + "@ai-sdk/provider-utils" "4.0.67" + "@vercel/oidc" "3.2.0" + "@ai-sdk/google-vertex@^4.0.68": version "4.0.169" resolved "https://registry.yarnpkg.com/@ai-sdk/google-vertex/-/google-vertex-4.0.169.tgz#9f73aca62e0ac808e80d40c553f8bbf3b8ec649b" @@ -90,6 +99,16 @@ "@standard-schema/spec" "^1.1.0" eventsource-parser "^3.0.8" +"@ai-sdk/provider-utils@4.0.67": + version "4.0.67" + resolved "https://registry.yarnpkg.com/@ai-sdk/provider-utils/-/provider-utils-4.0.67.tgz#b0fd45392b3726a5a63fcb1ab354bd377fcea9f8" + integrity sha512-0WV1upni0SUAKdsMTOADbPFU0CBji/XjNPcG0J+LWb5DZFtTWRrflj/oY7qi3zbJswvE1aWv/d+h/A8BEWnAWA== + dependencies: + "@ai-sdk/provider" "3.0.19" + "@standard-schema/spec" "^1.1.0" + eventsource-parser "^3.0.8" + undici "^6.28.0" + "@ai-sdk/provider@3.0.14": version "3.0.14" resolved "https://registry.yarnpkg.com/@ai-sdk/provider/-/provider-3.0.14.tgz#5893a90e0b5355ec6a5c148f06bd7f08355c1a6f" @@ -97,6 +116,13 @@ dependencies: json-schema "^0.4.0" +"@ai-sdk/provider@3.0.19": + version "3.0.19" + resolved "https://registry.yarnpkg.com/@ai-sdk/provider/-/provider-3.0.19.tgz#9adebbcb183f41d6f726f00efd528485b58d56d3" + integrity sha512-5lWq7zSfRaU5ylDESn3sDQjgGl2AI1vjZKO/6JY3aFW70ct4zcP1kN+ZUYOuTXr28fVsYbyu2YFhddLdIgQFZQ== + dependencies: + json-schema "^0.4.0" + "@ai-sdk/xai@^3.0.0": version "3.0.111" resolved "https://registry.yarnpkg.com/@ai-sdk/xai/-/xai-3.0.111.tgz#d83f0c6f76de3eb718487a72d181edfb6258a030" @@ -1917,6 +1943,16 @@ agent-base@^7.1.2: resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.4.tgz#e3cd76d4c548ee895d3c3fd8dc1f6c5b9032e7a8" integrity sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ== +ai@^6.0.0: + version "6.0.333" + resolved "https://registry.yarnpkg.com/ai/-/ai-6.0.333.tgz#bea2e2b78ce84729b8e4d31b6b33edf716fbf30a" + integrity sha512-mY+EpkQtTRHHJZ6CaKKJETsYmAEITKKPKmVY+FhJGUjDaLLl7CaF2Qgv8ShGN4eZgSSHbi34yfWx71QvQgKyOQ== + dependencies: + "@ai-sdk/gateway" "3.0.238" + "@ai-sdk/provider" "3.0.19" + "@ai-sdk/provider-utils" "4.0.67" + "@opentelemetry/api" "^1.9.0" + ai@^6.0.97: version "6.0.235" resolved "https://registry.yarnpkg.com/ai/-/ai-6.0.235.tgz#78d0836636fa06ac995cef501f6d0feb0d11ecb8" @@ -5207,6 +5243,11 @@ undici-types@~8.3.0: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-8.3.0.tgz#44e9fc9f3244648cdea35e4f9bb2d681e9410809" integrity sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ== +undici@^6.28.0: + version "6.29.1" + resolved "https://registry.yarnpkg.com/undici/-/undici-6.29.1.tgz#a3317c5df0236e0633727d9a7bd4f7ed195d8575" + integrity sha512-EvRlUt/QqAEvXdQeRMYUfwOQsfzpDztykrG2owGEE/rBOBoYv9uF39ZzsNDh4lTFHhTr2XOZjKkYgueD12ZmGA== + undici@^7.24.0: version "7.30.1" resolved "https://registry.yarnpkg.com/undici/-/undici-7.30.1.tgz#360b3de104b94abc52729aa1a4a07e027aa71f73" From 223ed4db1e66dc463eaf6620ffbee5d088c9e60f Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Wed, 18 Nov 2026 14:04:56 +0100 Subject: [PATCH 02/13] fix(evals): fail when a tool call fails, and run the deployer through lstk Every case only checked that the model called the right tool, so a case passed even when the tool returned an error. Each case now also matches a success marker inside a tool call's output. The tester matches against the whole simulation result as JSON, so each pattern starts with a prefix that keeps it inside one output string; otherwise the model's answer or the scenario could satisfy it. The spec requires every case to pass, and counts iterations lost to infrastructure errors (rate limits, 5xx) as failures instead of dropping them. The Cloud Pods case runs right after LocalStack starts, while a save is still quick enough for the tester's 60s call limit. Since #97 the deployer runs `lstk terraform`, so the workflow installs lstk instead of tflocal and downloads the sample project's providers up front. lstk and the Snowflake CLI come from the image's lockfiles; Terraform and uv from the Dockerfile's FROM lines, which a pin test now enforces for every workflow. Runs no longer overlap, because they share one Cloud Pod name, and the job timeout leaves room for setup and the upload, which now includes the results directory (upload-artifact skips dot directories by default). Drop the unused `ai` devDependency: the tester loads its own copy. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/mcp-evals.yml | 63 +++++++++++--- README.md | 1 + data/evals/comprehensive.json | 138 ++++++++++++++++++++----------- package.json | 1 - src/core/dependency-pins.test.ts | 31 ++++++- tests/mcp/evals.spec.mjs | 23 ++++-- yarn.lock | 41 --------- 7 files changed, 189 insertions(+), 109 deletions(-) diff --git a/.github/workflows/mcp-evals.yml b/.github/workflows/mcp-evals.yml index ca27f86..46f9bd4 100644 --- a/.github/workflows/mcp-evals.yml +++ b/.github/workflows/mcp-evals.yml @@ -14,35 +14,74 @@ on: permissions: contents: read +# Runs share the Cloud Pod name mcptester-e2e-pod in one LocalStack account, so they must not overlap. +concurrency: + group: mcp-evals + cancel-in-progress: false + jobs: mcp-evals: - runs-on: ubuntu-latest - # The spec itself times out after 90 minutes. - timeout-minutes: 100 + runs-on: ubuntu-26.04 + # The spec itself times out after 90 minutes; the rest covers setup and the results upload. + timeout-minutes: 120 env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} EVAL_MODEL: ${{ inputs.model }} steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Use Node.js 22 - uses: actions/setup-node@v4 + uses: actions/setup-node@v7 with: node-version: 22.x - name: Install dependencies - run: yarn + run: yarn install --frozen-lockfile + + # Terraform and uv run at the versions in the Dockerfile's FROM lines, which Dependabot + # keeps current, so the evals use the same tools as the image. + - name: Read tool versions from the Dockerfile + id: versions + run: | + terraform=$(sed -nE 's#^FROM hashicorp/terraform:([^ ]+).*#\1#p' Dockerfile) + uv=$(sed -nE 's#^FROM ghcr\.io/astral-sh/uv:([^ ]+).*#\1#p' Dockerfile) + # Separate commands: bash -e ignores a failure on the left of &&. + test -n "$terraform" + test -n "$uv" + echo "terraform=$terraform" >> "$GITHUB_OUTPUT" + echo "uv=$uv" >> "$GITHUB_OUTPUT" - # The deployer case runs tflocal; the wrapper would garble its output. + # The deployer case runs `lstk terraform`; the wrapper would garble its output. - name: Install Terraform - uses: hashicorp/setup-terraform@v3 + uses: hashicorp/setup-terraform@v4 with: + terraform_version: ${{ steps.versions.outputs.terraform }} terraform_wrapper: false - - name: Install tflocal and the Snowflake CLI - run: pipx install terraform-local && pipx install snowflake-cli + # The tester caps each tool call at the MCP SDK's 60s request timeout, and the deployer + # runs init, apply and output in one call. Download the AWS provider before the evals. + - name: Download the sample project's Terraform providers + run: terraform -chdir=data/sample-terraform init -input=false + + # lstk and the Snowflake CLI come from the same lockfiles as the Docker image. + - name: Install lstk + run: | + npm ci --prefix docker/tools/npm --omit=dev --ignore-scripts --no-audit --no-fund + echo "$GITHUB_WORKSPACE/docker/tools/npm/node_modules/.bin" >> "$GITHUB_PATH" + + - name: Install uv + uses: astral-sh/setup-uv@v10.3.3 + with: + version: ${{ steps.versions.outputs.uv }} + + - name: Install the Snowflake CLI + env: + UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/snowflake-cli + run: | + uv sync --project docker/tools/snowflake --locked --no-install-project --no-build + echo "$UV_PROJECT_ENVIRONMENT/bin" >> "$GITHUB_PATH" # Pulled up front so a start inside a case doesn't time out on the download. - name: Pull the LocalStack images @@ -55,8 +94,10 @@ jobs: - name: Keep the eval results if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: mcp-evals-results path: .mcp-test-results + # The directory starts with a dot, which upload-artifact skips by default. + include-hidden-files: true if-no-files-found: ignore diff --git a/README.md b/README.md index 40bd1ff..a65f181 100644 --- a/README.md +++ b/README.md @@ -268,6 +268,7 @@ This repository includes [MCP Server Tester](https://github.com/gleanwork/mcp-se export LOCALSTACK_AUTH_TOKEN="" yarn test:mcp:evals ``` + The evals start, change and stop LocalStack on port 4566, so don't run them against an instance you're using. They need Docker, Terraform, [lstk](https://github.com/localstack/lstk) and the Snowflake CLI (`snow`) on the `PATH`, and the sample project's providers downloaded with `terraform -chdir=data/sample-terraform init`. The [MCP evals workflow](./.github/workflows/mcp-evals.yml) runs them weekly and on demand; it needs the `OPENROUTER_API_KEY` repository secret. - Open the latest MCP Server Tester HTML report: ```bash diff --git a/data/evals/comprehensive.json b/data/evals/comprehensive.json index 5225343..7abba40 100644 --- a/data/evals/comprehensive.json +++ b/data/evals/comprehensive.json @@ -1,6 +1,6 @@ { "name": "localstack-mcp-comprehensive", - "description": "Scenario-based OpenRouter evals covering LocalStack MCP tools.", + "description": "Scenario-based OpenRouter evals covering LocalStack MCP tools. Each matchesPattern starts with a prefix that keeps it inside one tool call output: the tester matches against the whole simulation result as JSON, which also holds the scenario and the model answer.", "cases": [ { "id": "scenario-management-status", @@ -24,7 +24,49 @@ } ], "order": "any" - } + }, + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*Services initialized: [1-9]\\d*/\\d+"] + } + }, + { + "id": "scenario-cloud-pods-roundtrip", + "description": "localstack-cloud-pods: save, load, and delete a named pod. Runs second, right after the status case starts LocalStack: the eval host times out each tool call at ~60s (MCP SDK default; the tester's requestTimeoutMs is not applied), and a save takes longer the more state later cases add.", + "mode": "mcp_host", + "scenario": "Using the LocalStack Cloud Pods tool, save the current LocalStack state to a remote Cloud Pod named mcptester-e2e-pod. If saving fails because a pod with that name already exists, delete the existing pod and save again. If a save, load, or delete attempt times out, retry that same operation once before giving up. After saving, load the Cloud Pod mcptester-e2e-pod back into LocalStack. Finally, delete the Cloud Pod mcptester-e2e-pod so nothing is left behind.", + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0, + "maxToolCalls": 15 + }, + "iterations": 5, + "accuracyThreshold": 0.8, + "expect": { + "toolsTriggered": { + "calls": [ + { + "name": "localstack-cloud-pods", + "arguments": { "action": "save", "pod_name": "mcptester-e2e-pod" }, + "required": true + }, + { + "name": "localstack-cloud-pods", + "arguments": { "action": "load", "pod_name": "mcptester-e2e-pod" }, + "required": true + }, + { + "name": "localstack-cloud-pods", + "arguments": { "action": "delete", "pod_name": "mcptester-e2e-pod" }, + "required": true + } + ], + "order": "strict" + }, + "matchesPattern": [ + "\"output\":\"(?:[^\"\\\\]|\\\\.)*was saved successfully", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*was loaded\\. Your LocalStack instance has been restored", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*has been permanently deleted" + ] } }, { @@ -45,7 +87,8 @@ { "name": "localstack-deployer", "arguments": { "action": "deploy" }, "required": true } ], "order": "any" - } + }, + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*Apply complete!"] } }, { @@ -70,7 +113,8 @@ } ], "order": "any" - } + }, + "matchesPattern": ["^(?![\\s\\S]*\"output\":\"(?:[^\"\\\\]|\\\\.)*❌)"] } }, { @@ -95,7 +139,10 @@ } ], "order": "any" - } + }, + "matchesPattern": [ + "\"output\":\"(?:[^\"\\\\]|\\\\.)*(?:# 🚨 LocalStack Errors|βœ… No errors found)" + ] } }, { @@ -125,7 +172,11 @@ } ], "order": "any" - } + }, + "matchesPattern": [ + "\"output\":\"(?:[^\"\\\\]|\\\\.)*## App Inspector Status(?! Updated)", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*## Traces" + ] } }, { @@ -155,7 +206,11 @@ } ], "order": "any" - } + }, + "matchesPattern": [ + "\"output\":\"(?:[^\"\\\\]|\\\\.)*\\*\\*LocalStack IAM Enforcement Status\\*\\*", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*IAM enforcement mode has been set to `SOFT_MODE`" + ] } }, { @@ -180,43 +235,8 @@ } ], "order": "any" - } - } - }, - { - "id": "scenario-cloud-pods-roundtrip", - "description": "localstack-cloud-pods: save, load, and delete a named pod. NOTE: the eval host times out individual tool calls at ~60s (MCP SDK default; the tester's requestTimeoutMs is not applied), so this case needs a reasonably fresh container whose state saves quickly.", - "mode": "mcp_host", - "scenario": "Using the LocalStack Cloud Pods tool, save the current LocalStack state to a remote Cloud Pod named mcptester-e2e-pod. If saving fails because a pod with that name already exists, delete the existing pod and save again. If a save, load, or delete attempt times out, retry that same operation once before giving up. After saving, load the Cloud Pod mcptester-e2e-pod back into LocalStack. Finally, delete the Cloud Pod mcptester-e2e-pod so nothing is left behind.", - "mcpHostConfig": { - "provider": "openrouter", - "model": "google/gemini-2.5-flash", - "temperature": 0, - "maxToolCalls": 15 - }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-cloud-pods", - "arguments": { "action": "save", "pod_name": "mcptester-e2e-pod" }, - "required": true - }, - { - "name": "localstack-cloud-pods", - "arguments": { "action": "load", "pod_name": "mcptester-e2e-pod" }, - "required": true - }, - { - "name": "localstack-cloud-pods", - "arguments": { "action": "delete", "pod_name": "mcptester-e2e-pod" }, - "required": true - } - ], - "order": "strict" - } + }, + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*## LocalStack State Inspect"] } }, { @@ -251,7 +271,12 @@ } ], "order": "strict" - } + }, + "matchesPattern": [ + "\"output\":\"(?:[^\"\\\\]|\\\\.)*Latency of 100ms has been injected", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*The current network latency is 100ms", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*Network latency has been cleared" + ] } }, { @@ -276,7 +301,10 @@ } ], "order": "any" - } + }, + "matchesPattern": [ + "\"output\":\"(?:[^\"\\\\]|\\\\.)*\\d+ extensions available\\. Install any" + ] } }, { @@ -306,7 +334,11 @@ } ], "order": "any" - } + }, + "matchesPattern": [ + "\"output\":\"(?:[^\"\\\\]|\\\\.)*## AWS Replicator Supported Resources", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*## AWS Replicator Jobs" + ] } }, { @@ -331,7 +363,8 @@ } ], "order": "any" - } + }, + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*## Ephemeral Instances"] } }, { @@ -350,7 +383,8 @@ "toolsTriggered": { "calls": [{ "name": "localstack-docs", "required": true }], "order": "any" - } + }, + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*\\*\\*[1-9]\\d* results? found\\*\\*"] } }, { @@ -386,7 +420,11 @@ } ], "order": "strict" - } + }, + "matchesPattern": [ + "\"output\":\"(?:[^\"\\\\]|\\\\.)*(?:\\| Status\\s+\\| OK\\s+\\|)", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*(?:\\| CUSTOMERS\\s+\\|)" + ] } } ] diff --git a/package.json b/package.json index 25bbd1d..c8d202b 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,6 @@ "@types/dockerode": "^4.0.2", "@types/jest": "^30.0.0", "@xmcp-dev/compiler": "^1.9.0", - "ai": "^6.0.0", "audit-ci": "^7.1.0", "esbuild": "^0.28.2", "eslint-config-prettier": "^10.1.8", diff --git a/src/core/dependency-pins.test.ts b/src/core/dependency-pins.test.ts index 84b0d47..c8c93fd 100644 --- a/src/core/dependency-pins.test.ts +++ b/src/core/dependency-pins.test.ts @@ -78,7 +78,11 @@ describe("dependency pins", () => { // Runners are either a literal label or `${{ matrix.runner }}`, whose values are checked // as literals; any other expression could hide an unpinned label. const docker = read(".github/workflows/docker.yml"); - const workflow = [docker, read(".github/workflows/ci.yml")].join("\n"); + const workflow = [ + docker, + read(".github/workflows/ci.yml"), + read(".github/workflows/mcp-evals.yml"), + ].join("\n"); for (const [, value] of workflow.matchAll(/^[ \t]*runs-on:[ \t]*(.*)$/gm)) { if (value.startsWith("${{")) expect(value).toBe("${{ matrix.runner }}"); } @@ -93,6 +97,31 @@ describe("dependency pins", () => { expect(docker).toMatch(/^[ \t]*(?:-[ \t]*)?runner:[ \t]*ubuntu-[\d.]+-arm[ \t]*$/m); }); + it("takes Terraform and uv versions in workflows from the Dockerfile's FROM lines", () => { + // Dependabot bumps the FROM lines but not action inputs, so a literal version would drift. + const dockerfile = read("Dockerfile"); + expect(dockerfile).toMatch(/^FROM hashicorp\/terraform:\S+/m); + expect(dockerfile).toMatch(/^FROM ghcr\.io\/astral-sh\/uv:\S+/m); + + // Without its version input, each action installs the latest release. Steps are split at + // their leading "- ", so a step's inputs may come in any order. + const inputs = { + "hashicorp/setup-terraform": "terraform_version", + "astral-sh/setup-uv": "version", + }; + for (const file of fs.readdirSync(path.join(ROOT, ".github/workflows"))) { + for (const step of read(`.github/workflows/${file}`).split(/^[ \t]*- /m)) { + for (const [action, input] of Object.entries(inputs)) { + if (!new RegExp(`uses:[ \\t]*${action}@`).test(step)) continue; + const version = step.match(new RegExp(`^[ \\t]*${input}:[ \\t]*(.*)$`, "m"))?.[1]; + expect(`${file} ${action}: ${version}`).toMatch( + /: \$\{\{ steps\.\w+\.outputs\.\w+ \}\}$/ + ); + } + } + } + }); + it("installs CI dependencies from the same frozen lockfile as the image", () => { for (const file of fs.readdirSync(path.join(ROOT, ".github/workflows"))) { for (const match of read(`.github/workflows/${file}`).matchAll(/run:\s*(yarn\b.*)/g)) { diff --git a/tests/mcp/evals.spec.mjs b/tests/mcp/evals.spec.mjs index 6a368dc..8f8316b 100644 --- a/tests/mcp/evals.spec.mjs +++ b/tests/mcp/evals.spec.mjs @@ -26,18 +26,31 @@ test.describe("OpenRouter comprehensive eval", () => { } const result = await runEvalDataset({ dataset }, { mcp, testInfo }); const caseResults = result.caseResults || []; - const passed = caseResults.filter((entry) => entry?.pass === true).length; - const failed = caseResults.filter((entry) => entry?.pass !== true); + // The tester leaves iterations with infrastructure errors (rate limits, 5xx, network) out of + // the pass rate, so 1 good iteration out of 5 can pass a case. Count them as failures instead. + const thresholds = new Map( + dataset.cases.map((c) => [c.id, Math.ceil((c.accuracyThreshold ?? 1) * (c.iterations ?? 1))]) + ); + const passesCase = (entry) => + entry?.pass === true && + (entry.iterationResults ?? [entry]).filter((r) => r.pass === true).length >= + thresholds.get(entry.id); + const passed = caseResults.filter(passesCase).length; + const failed = caseResults.filter((entry) => !passesCase(entry)); if (failed.length > 0) { for (const entry of failed) { - console.error(`Eval case failed: ${entry.id}`); + console.error( + `Eval case failed: ${entry.id} (infrastructure errors: ${entry.infrastructureErrorCount ?? 0})` + ); console.error(JSON.stringify(entry, null, 2)); } } console.log(`Eval pass rate: ${passed}/${caseResults.length}`); - const passRate = caseResults.length > 0 ? passed / caseResults.length : 1; - expect(passRate).toBeGreaterThanOrEqual(0.75); + // Every case must pass. A case's own accuracyThreshold already absorbs flaky iterations, so a + // case that fails here failed most of them, e.g. because its tool is broken. + expect(caseResults).toHaveLength(dataset.cases.length); + expect(failed.map((entry) => entry.id)).toEqual([]); }); }); diff --git a/yarn.lock b/yarn.lock index ac2aaa5..443a457 100644 --- a/yarn.lock +++ b/yarn.lock @@ -37,15 +37,6 @@ "@ai-sdk/provider-utils" "4.0.40" "@vercel/oidc" "3.2.0" -"@ai-sdk/gateway@3.0.238": - version "3.0.238" - resolved "https://registry.yarnpkg.com/@ai-sdk/gateway/-/gateway-3.0.238.tgz#b1465e928ad7b24925afbcdeaf09197b64e1c8a7" - integrity sha512-Kqdof17ZXdaKaHsoY/Z4HSbji86DVbIaG0kSgQwrKlEQIfp9Aip2S6xbba063EtLKWx0m/psvzCBYP73YUyWKA== - dependencies: - "@ai-sdk/provider" "3.0.19" - "@ai-sdk/provider-utils" "4.0.67" - "@vercel/oidc" "3.2.0" - "@ai-sdk/google-vertex@^4.0.68": version "4.0.169" resolved "https://registry.yarnpkg.com/@ai-sdk/google-vertex/-/google-vertex-4.0.169.tgz#9f73aca62e0ac808e80d40c553f8bbf3b8ec649b" @@ -99,16 +90,6 @@ "@standard-schema/spec" "^1.1.0" eventsource-parser "^3.0.8" -"@ai-sdk/provider-utils@4.0.67": - version "4.0.67" - resolved "https://registry.yarnpkg.com/@ai-sdk/provider-utils/-/provider-utils-4.0.67.tgz#b0fd45392b3726a5a63fcb1ab354bd377fcea9f8" - integrity sha512-0WV1upni0SUAKdsMTOADbPFU0CBji/XjNPcG0J+LWb5DZFtTWRrflj/oY7qi3zbJswvE1aWv/d+h/A8BEWnAWA== - dependencies: - "@ai-sdk/provider" "3.0.19" - "@standard-schema/spec" "^1.1.0" - eventsource-parser "^3.0.8" - undici "^6.28.0" - "@ai-sdk/provider@3.0.14": version "3.0.14" resolved "https://registry.yarnpkg.com/@ai-sdk/provider/-/provider-3.0.14.tgz#5893a90e0b5355ec6a5c148f06bd7f08355c1a6f" @@ -116,13 +97,6 @@ dependencies: json-schema "^0.4.0" -"@ai-sdk/provider@3.0.19": - version "3.0.19" - resolved "https://registry.yarnpkg.com/@ai-sdk/provider/-/provider-3.0.19.tgz#9adebbcb183f41d6f726f00efd528485b58d56d3" - integrity sha512-5lWq7zSfRaU5ylDESn3sDQjgGl2AI1vjZKO/6JY3aFW70ct4zcP1kN+ZUYOuTXr28fVsYbyu2YFhddLdIgQFZQ== - dependencies: - json-schema "^0.4.0" - "@ai-sdk/xai@^3.0.0": version "3.0.111" resolved "https://registry.yarnpkg.com/@ai-sdk/xai/-/xai-3.0.111.tgz#d83f0c6f76de3eb718487a72d181edfb6258a030" @@ -1943,16 +1917,6 @@ agent-base@^7.1.2: resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.4.tgz#e3cd76d4c548ee895d3c3fd8dc1f6c5b9032e7a8" integrity sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ== -ai@^6.0.0: - version "6.0.333" - resolved "https://registry.yarnpkg.com/ai/-/ai-6.0.333.tgz#bea2e2b78ce84729b8e4d31b6b33edf716fbf30a" - integrity sha512-mY+EpkQtTRHHJZ6CaKKJETsYmAEITKKPKmVY+FhJGUjDaLLl7CaF2Qgv8ShGN4eZgSSHbi34yfWx71QvQgKyOQ== - dependencies: - "@ai-sdk/gateway" "3.0.238" - "@ai-sdk/provider" "3.0.19" - "@ai-sdk/provider-utils" "4.0.67" - "@opentelemetry/api" "^1.9.0" - ai@^6.0.97: version "6.0.235" resolved "https://registry.yarnpkg.com/ai/-/ai-6.0.235.tgz#78d0836636fa06ac995cef501f6d0feb0d11ecb8" @@ -5243,11 +5207,6 @@ undici-types@~8.3.0: resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-8.3.0.tgz#44e9fc9f3244648cdea35e4f9bb2d681e9410809" integrity sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ== -undici@^6.28.0: - version "6.29.1" - resolved "https://registry.yarnpkg.com/undici/-/undici-6.29.1.tgz#a3317c5df0236e0633727d9a7bd4f7ed195d8575" - integrity sha512-EvRlUt/QqAEvXdQeRMYUfwOQsfzpDztykrG2owGEE/rBOBoYv9uF39ZzsNDh4lTFHhTr2XOZjKkYgueD12ZmGA== - undici@^7.24.0: version "7.30.1" resolved "https://registry.yarnpkg.com/undici/-/undici-7.30.1.tgz#360b3de104b94abc52729aa1a4a07e027aa71f73" From fb1ab16f66c99aa3c881c3b164f5d53993601d0d Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Thu, 19 Nov 2026 10:32:07 +0100 Subject: [PATCH 03/13] ci: copy Terraform and uv out of the Dockerfile instead of parsing it The evals workflow read the Terraform and uv versions out of the Dockerfile's FROM lines with sed and passed them to setup actions. A new `host-tools` stage copies the binaries from the same stages the image uses, and a composite action (.github/actions/host-tools) builds it, installs lstk and optionally the Snowflake CLI from the image's lockfiles, and puts them on the PATH. The Dockerfile stays the one place Dependabot bumps these versions, and every job that needs the tools uses the action. The pin test now fails when a workflow installs Terraform or uv any other way. Co-Authored-By: Claude Opus 5.5 --- .github/actions/host-tools/action.yml | 33 ++++++++++++++++++++ .github/workflows/mcp-evals.yml | 44 ++++----------------------- Dockerfile | 6 ++++ src/core/dependency-pins.test.ts | 38 ++++++++++------------- 4 files changed, 61 insertions(+), 60 deletions(-) create mode 100644 .github/actions/host-tools/action.yml diff --git a/.github/actions/host-tools/action.yml b/.github/actions/host-tools/action.yml new file mode 100644 index 0000000..83a466d --- /dev/null +++ b/.github/actions/host-tools/action.yml @@ -0,0 +1,33 @@ +name: Host tools +description: >- + Puts the Docker image's tools on the PATH for jobs that run the MCP server on the host: + Terraform and uv from the Dockerfile's host-tools stage, lstk from docker/tools/npm, and + optionally the Snowflake CLI from docker/tools/snowflake. Versions change only in those files. +inputs: + snowflake: + description: Also install the Snowflake CLI. + required: false + default: "false" +runs: + using: composite + steps: + - name: Copy Terraform and uv out of the Dockerfile + shell: bash + run: | + docker build --target host-tools --output "type=local,dest=$RUNNER_TEMP/host-tools" . + echo "$RUNNER_TEMP/host-tools" >> "$GITHUB_PATH" + + - name: Install lstk + shell: bash + run: | + npm ci --prefix docker/tools/npm --omit=dev --ignore-scripts --no-audit --no-fund + echo "$GITHUB_WORKSPACE/docker/tools/npm/node_modules/.bin" >> "$GITHUB_PATH" + + - name: Install the Snowflake CLI + if: inputs.snowflake == 'true' + shell: bash + env: + UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/snowflake-cli + run: | + "$RUNNER_TEMP/host-tools/uv" sync --project docker/tools/snowflake --locked --no-install-project --no-build + echo "$UV_PROJECT_ENVIRONMENT/bin" >> "$GITHUB_PATH" diff --git a/.github/workflows/mcp-evals.yml b/.github/workflows/mcp-evals.yml index 46f9bd4..4c2700e 100644 --- a/.github/workflows/mcp-evals.yml +++ b/.github/workflows/mcp-evals.yml @@ -40,49 +40,17 @@ jobs: - name: Install dependencies run: yarn install --frozen-lockfile - # Terraform and uv run at the versions in the Dockerfile's FROM lines, which Dependabot - # keeps current, so the evals use the same tools as the image. - - name: Read tool versions from the Dockerfile - id: versions - run: | - terraform=$(sed -nE 's#^FROM hashicorp/terraform:([^ ]+).*#\1#p' Dockerfile) - uv=$(sed -nE 's#^FROM ghcr\.io/astral-sh/uv:([^ ]+).*#\1#p' Dockerfile) - # Separate commands: bash -e ignores a failure on the left of &&. - test -n "$terraform" - test -n "$uv" - echo "terraform=$terraform" >> "$GITHUB_OUTPUT" - echo "uv=$uv" >> "$GITHUB_OUTPUT" - - # The deployer case runs `lstk terraform`; the wrapper would garble its output. - - name: Install Terraform - uses: hashicorp/setup-terraform@v4 + # Terraform, lstk and the Snowflake CLI at the Docker image's versions. + - name: Install the image's tools + uses: ./.github/actions/host-tools with: - terraform_version: ${{ steps.versions.outputs.terraform }} - terraform_wrapper: false + snowflake: true - # The tester caps each tool call at the MCP SDK's 60s request timeout, and the deployer - # runs init, apply and output in one call. Download the AWS provider before the evals. + # The eval host caps each tool call at the MCP SDK's 60s request timeout, and the + # deployer runs init, apply and output in one call. Download the AWS provider first. - name: Download the sample project's Terraform providers run: terraform -chdir=data/sample-terraform init -input=false - # lstk and the Snowflake CLI come from the same lockfiles as the Docker image. - - name: Install lstk - run: | - npm ci --prefix docker/tools/npm --omit=dev --ignore-scripts --no-audit --no-fund - echo "$GITHUB_WORKSPACE/docker/tools/npm/node_modules/.bin" >> "$GITHUB_PATH" - - - name: Install uv - uses: astral-sh/setup-uv@v10.3.3 - with: - version: ${{ steps.versions.outputs.uv }} - - - name: Install the Snowflake CLI - env: - UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/snowflake-cli - run: | - uv sync --project docker/tools/snowflake --locked --no-install-project --no-build - echo "$UV_PROJECT_ENVIRONMENT/bin" >> "$GITHUB_PATH" - # Pulled up front so a start inside a case doesn't time out on the download. - name: Pull the LocalStack images run: | diff --git a/Dockerfile b/Dockerfile index 9f297f5..8f37794 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,12 @@ FROM hashicorp/terraform:1.17.2 AS terraform # uv installs the Python CLIs below; it is mounted at build time and not part of the image. FROM ghcr.io/astral-sh/uv:0.13.9 AS uv +# Terraform and uv for CI jobs that run on the host, at the image's versions +# (.github/actions/host-tools): docker build --target host-tools --output type=local,dest=DIR . +FROM scratch AS host-tools +COPY --from=terraform /bin/terraform / +COPY --from=uv /uv /uvx / + FROM node:22.23.5-bookworm-slim AS runtime ENV DEBIAN_FRONTEND=noninteractive \ PYTHONDONTWRITEBYTECODE=1 \ diff --git a/src/core/dependency-pins.test.ts b/src/core/dependency-pins.test.ts index c8c93fd..ce9c627 100644 --- a/src/core/dependency-pins.test.ts +++ b/src/core/dependency-pins.test.ts @@ -97,28 +97,22 @@ describe("dependency pins", () => { expect(docker).toMatch(/^[ \t]*(?:-[ \t]*)?runner:[ \t]*ubuntu-[\d.]+-arm[ \t]*$/m); }); - it("takes Terraform and uv versions in workflows from the Dockerfile's FROM lines", () => { - // Dependabot bumps the FROM lines but not action inputs, so a literal version would drift. - const dockerfile = read("Dockerfile"); - expect(dockerfile).toMatch(/^FROM hashicorp\/terraform:\S+/m); - expect(dockerfile).toMatch(/^FROM ghcr\.io\/astral-sh\/uv:\S+/m); - - // Without its version input, each action installs the latest release. Steps are split at - // their leading "- ", so a step's inputs may come in any order. - const inputs = { - "hashicorp/setup-terraform": "terraform_version", - "astral-sh/setup-uv": "version", - }; - for (const file of fs.readdirSync(path.join(ROOT, ".github/workflows"))) { - for (const step of read(`.github/workflows/${file}`).split(/^[ \t]*- /m)) { - for (const [action, input] of Object.entries(inputs)) { - if (!new RegExp(`uses:[ \\t]*${action}@`).test(step)) continue; - const version = step.match(new RegExp(`^[ \\t]*${input}:[ \\t]*(.*)$`, "m"))?.[1]; - expect(`${file} ${action}: ${version}`).toMatch( - /: \$\{\{ steps\.\w+\.outputs\.\w+ \}\}$/ - ); - } - } + it("gives CI jobs the image's Terraform and uv, from the Dockerfile's host-tools stage", () => { + // Dependabot bumps the FROM lines, not action inputs or install scripts, so any other way + // of installing these tools in CI would drift from the image. + const stage = dockerfileInstructions().join("\n"); + expect(stage).toMatch(/^FROM scratch AS host-tools$/m); + expect(stage).toMatch(/^COPY --from=terraform \S+ \/$/m); + expect(stage).toMatch(/^COPY --from=uv \S+ \S+ \/$/m); + + const files = [ + ...fs.readdirSync(path.join(ROOT, ".github/workflows")).map((f) => `.github/workflows/${f}`), + ".github/actions/host-tools/action.yml", + ]; + for (const file of files) { + expect( + `${file}: ${read(file).match(/setup-terraform|setup-uv|terraform_version|releases\.hashicorp\.com|astral\.sh\/uv/g) ?? []}` + ).toBe(`${file}: `); } }); From 710274ac008bfa1fd8c10521deffe801d67fdeaf Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Thu, 19 Nov 2026 10:32:07 +0100 Subject: [PATCH 04/13] test: check each tool without a model, and evals for tool selection only tests/mcp/tools.spec.mjs calls each tool against a running LocalStack in a fixed order (start, deploy, list, inspect, IAM, chaos, Cloud Pods, replicator, extensions, ephemeral instances) and checks text only a successful result contains. It needs no model, so a new CI job runs it on every pull request. It calls the SDK client directly with a 5-minute timeout: the tester's callTool applies the SDK's 60s default, which a start or deploy can exceed. With the tools covered there, the evals only check whether a model picks and combines the right tools. Their prompts now describe a goal instead of naming the tool, the cases that only repeated a single tool call moved to the tool tests, and each case runs once: 9 model runs instead of 68. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 29 +++++ README.md | 10 +- data/evals/comprehensive.json | 225 +++++----------------------------- package.json | 3 +- tests/mcp/tools.spec.mjs | 145 ++++++++++++++++++++++ 5 files changed, 213 insertions(+), 199 deletions(-) create mode 100644 tests/mcp/tools.spec.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02ea70e..20c159c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,3 +56,32 @@ jobs: run: yarn test:mcp:direct env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + + # Each tool against a real LocalStack, without a model (tests/mcp/tools.spec.mjs). The first + # test starts LocalStack through the management tool. + mcp-tool-tests: + runs-on: ubuntu-26.04 + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Use Node.js 22 + uses: actions/setup-node@v7 + with: + node-version: 22.x + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Install the image's tools + uses: ./.github/actions/host-tools + + # Pulled up front so the start stays within the management tool's 120s. + - name: Pull the LocalStack image + run: docker pull localstack/localstack-pro:latest + + - name: Run the tool tests + run: yarn test:mcp:tools diff --git a/README.md b/README.md index a65f181..9ed7384 100644 --- a/README.md +++ b/README.md @@ -262,19 +262,25 @@ This repository includes [MCP Server Tester](https://github.com/gleanwork/mcp-se ```bash yarn test:mcp:direct ``` +- Run each tool against a running LocalStack, without a model (deterministic; starts LocalStack if it isn't running): + ```bash + export LOCALSTACK_AUTH_TOKEN="" + yarn test:mcp:tools + ``` + The tests deploy a sample project, switch IAM enforcement to soft mode and save a Cloud Pod, so point them at an instance you don't need: set `LOCALSTACK_PORT` and `MAIN_CONTAINER_NAME` to use one other than the default on 4566. They need Docker, Terraform and [lstk](https://github.com/localstack/lstk) on the `PATH`. - Run MCP host evals through [OpenRouter](https://openrouter.ai) (default model `google/gemini-2.5-flash`; set `EVAL_MODEL` to any OpenRouter model ID to override it): ```bash export OPENROUTER_API_KEY="" export LOCALSTACK_AUTH_TOKEN="" yarn test:mcp:evals ``` - The evals start, change and stop LocalStack on port 4566, so don't run them against an instance you're using. They need Docker, Terraform, [lstk](https://github.com/localstack/lstk) and the Snowflake CLI (`snow`) on the `PATH`, and the sample project's providers downloaded with `terraform -chdir=data/sample-terraform init`. + The evals check whether a model picks and combines the right tools for a goal; their prompts don't name the tools. They start, change and stop LocalStack on port 4566, so don't run them against an instance you're using. They need Docker, Terraform, [lstk](https://github.com/localstack/lstk) and the Snowflake CLI (`snow`) on the `PATH`, and the sample project's providers downloaded with `terraform -chdir=data/sample-terraform init`. The [MCP evals workflow](./.github/workflows/mcp-evals.yml) runs them weekly and on demand; it needs the `OPENROUTER_API_KEY` repository secret. - Open the latest MCP Server Tester HTML report: ```bash npx mcp-server-tester open ``` -- Run both: +- Run all of them: ```bash yarn test:mcp ``` diff --git a/data/evals/comprehensive.json b/data/evals/comprehensive.json index 7abba40..47cd2cd 100644 --- a/data/evals/comprehensive.json +++ b/data/evals/comprehensive.json @@ -1,25 +1,22 @@ { "name": "localstack-mcp-comprehensive", - "description": "Scenario-based OpenRouter evals covering LocalStack MCP tools. Each matchesPattern starts with a prefix that keeps it inside one tool call output: the tester matches against the whole simulation result as JSON, which also holds the scenario and the model answer.", + "description": "Whether a model picks and combines the LocalStack MCP tools; tests/mcp/tools.spec.mjs checks each tool without a model. Prompts describe the goal, not the tool. One iteration per case. Each matchesPattern starts with a prefix that keeps it inside one tool call output: the tester matches against the whole simulation result as JSON, which also holds the scenario and the model answer.", "cases": [ { "id": "scenario-management-status", - "description": "localstack-management: status check of the AWS emulator.", + "description": "localstack-management: status, and a start when LocalStack isn't running. Runs first, so later cases have a running LocalStack.", "mode": "mcp_host", - "scenario": "Check the current status of the LocalStack AWS emulator using the LocalStack management tool. If it is not running, start it. Report whether it is ready to accept requests.", + "scenario": "Is my local AWS emulator running? If it isn't, bring it up. Tell me once it is ready to accept requests.", "mcpHostConfig": { "provider": "openrouter", "model": "google/gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, "expect": { "toolsTriggered": { "calls": [ { "name": "localstack-management", - "arguments": { "action": "status" }, "required": true } ], @@ -30,17 +27,15 @@ }, { "id": "scenario-cloud-pods-roundtrip", - "description": "localstack-cloud-pods: save, load, and delete a named pod. Runs second, right after the status case starts LocalStack: the eval host times out each tool call at ~60s (MCP SDK default; the tester's requestTimeoutMs is not applied), and a save takes longer the more state later cases add.", + "description": "localstack-cloud-pods: save, load and delete a named pod. Runs second, right after LocalStack starts: the eval host times out each tool call at ~60s (MCP SDK default; the tester's requestTimeoutMs is not applied), and a save takes longer the more state later cases add.", "mode": "mcp_host", - "scenario": "Using the LocalStack Cloud Pods tool, save the current LocalStack state to a remote Cloud Pod named mcptester-e2e-pod. If saving fails because a pod with that name already exists, delete the existing pod and save again. If a save, load, or delete attempt times out, retry that same operation once before giving up. After saving, load the Cloud Pod mcptester-e2e-pod back into LocalStack. Finally, delete the Cloud Pod mcptester-e2e-pod so nothing is left behind.", + "scenario": "Save the current state of my local AWS emulator as a remote snapshot named mcptester-e2e-pod that a teammate could restore. If a snapshot with that name already exists, delete it and save again. Then restore mcptester-e2e-pod into my emulator to check that it works, and finally delete the snapshot so nothing is left behind. If a save, restore or delete times out, retry it once before giving up.", "mcpHostConfig": { "provider": "openrouter", "model": "google/gemini-2.5-flash", "temperature": 0, "maxToolCalls": 15 }, - "iterations": 5, - "accuracyThreshold": 0.8, "expect": { "toolsTriggered": { "calls": [ @@ -71,134 +66,64 @@ }, { "id": "scenario-deploy-terraform-project", - "description": "localstack-deployer: deploy the sample Terraform project.", + "description": "localstack-deployer: deploy the sample Terraform project, which creates the bucket the next case lists.", "mode": "mcp_host", - "scenario": "Deploy the Terraform project located in the data/sample-terraform directory of this repository to LocalStack using the LocalStack deployer tool, then report the Terraform outputs.", + "scenario": "Deploy the Terraform project in this repository's data/sample-terraform directory to my local AWS emulator, then tell me its outputs.", "mcpHostConfig": { "provider": "openrouter", "model": "google/gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { "name": "localstack-deployer", "arguments": { "action": "deploy" }, "required": true } - ], - "order": "any" - }, - "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*Apply complete!"] - } - }, - { - "id": "scenario-aws-client-list-buckets", - "description": "localstack-aws-client: run a pinned AWS CLI command.", - "mode": "mcp_host", - "scenario": "Using the LocalStack AWS client tool, run the AWS CLI command \"s3 ls\" against LocalStack. Pass the command string exactly as s3 ls, with no aws or awslocal prefix and no extra flags. Then tell me which S3 buckets exist.", - "mcpHostConfig": { - "provider": "openrouter", - "model": "google/gemini-2.5-flash", - "temperature": 0 - }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-aws-client", - "arguments": { "command": "s3 ls" }, - "required": true - } - ], - "order": "any" - }, - "matchesPattern": ["^(?![\\s\\S]*\"output\":\"(?:[^\"\\\\]|\\\\.)*❌)"] - } - }, - { - "id": "scenario-logs-analysis-errors", - "description": "localstack-logs-analysis: errors analysis of container logs.", - "mode": "mcp_host", - "scenario": "Use the LocalStack logs analysis tool to run an 'errors' analysis of the recent LocalStack container logs and report which AWS API calls failed.", - "mcpHostConfig": { - "provider": "openrouter", - "model": "google/gemini-2.5-flash", - "temperature": 0 - }, - "iterations": 5, - "accuracyThreshold": 0.8, "expect": { "toolsTriggered": { "calls": [ { - "name": "localstack-logs-analysis", - "arguments": { "analysisType": "errors" }, + "name": "localstack-deployer", + "arguments": { "action": "deploy" }, "required": true } ], "order": "any" }, - "matchesPattern": [ - "\"output\":\"(?:[^\"\\\\]|\\\\.)*(?:# 🚨 LocalStack Errors|βœ… No errors found)" - ] + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*Apply complete!"] } }, { - "id": "scenario-app-inspector-traces", - "description": "localstack-app-inspector: status check plus trace listing.", + "id": "scenario-aws-client-list-buckets", + "description": "localstack-aws-client: list S3 buckets, which include the one the deploy case created.", "mode": "mcp_host", - "scenario": "Using the LocalStack App Inspector tool, first check whether App Inspector is currently enabled, enable it only if it is disabled, and then list the most recent App Inspector traces.", + "scenario": "Which S3 buckets exist in my local AWS emulator right now?", "mcpHostConfig": { "provider": "openrouter", "model": "google/gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, "expect": { "toolsTriggered": { "calls": [ { - "name": "localstack-app-inspector", - "arguments": { "action": "get-status" }, - "required": true - }, - { - "name": "localstack-app-inspector", - "arguments": { "action": "list-traces" }, + "name": "localstack-aws-client", "required": true } ], "order": "any" }, - "matchesPattern": [ - "\"output\":\"(?:[^\"\\\\]|\\\\.)*## App Inspector Status(?! Updated)", - "\"output\":\"(?:[^\"\\\\]|\\\\.)*## Traces" - ] + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*mcpjam-eval-bucket"] } }, { "id": "scenario-iam-soft-mode", - "description": "localstack-iam-policy-analyzer: read mode, then set SOFT_MODE.", + "description": "localstack-iam-policy-analyzer: switch IAM enforcement to soft mode from a symptom, not the mode's name.", "mode": "mcp_host", - "scenario": "Check the current IAM enforcement mode in LocalStack using the IAM policy analyzer, then set the enforcement mode to SOFT_MODE with the set-mode action. Perform the set-mode call even if the current mode is already SOFT_MODE, so the setting is explicitly confirmed.", + "scenario": "My Lambda function keeps failing with AccessDenied errors in my local AWS emulator. Instead of blocking those calls, let them through and only log which permissions are missing.", "mcpHostConfig": { "provider": "openrouter", "model": "google/gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, "expect": { "toolsTriggered": { "calls": [ - { - "name": "localstack-iam-policy-analyzer", - "arguments": { "action": "get-status" }, - "required": true - }, { "name": "localstack-iam-policy-analyzer", "arguments": { "action": "set-mode", "mode": "SOFT_MODE" }, @@ -208,49 +133,20 @@ "order": "any" }, "matchesPattern": [ - "\"output\":\"(?:[^\"\\\\]|\\\\.)*\\*\\*LocalStack IAM Enforcement Status\\*\\*", "\"output\":\"(?:[^\"\\\\]|\\\\.)*IAM enforcement mode has been set to `SOFT_MODE`" ] } }, - { - "id": "scenario-state-inspect", - "description": "localstack-state-management: read-only inspect of runtime state.", - "mode": "mcp_host", - "scenario": "Using the LocalStack state management tool, inspect the in-memory state of the running LocalStack instance and tell me which AWS services currently hold state. Do not export, import, or reset anything.", - "mcpHostConfig": { - "provider": "openrouter", - "model": "google/gemini-2.5-flash", - "temperature": 0 - }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-state-management", - "arguments": { "action": "inspect" }, - "required": true - } - ], - "order": "any" - }, - "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*## LocalStack State Inspect"] - } - }, { "id": "scenario-chaos-latency-cycle", - "description": "localstack-chaos-injector: inject, verify, and clear latency.", + "description": "localstack-chaos-injector: inject, read back and clear network latency.", "mode": "mcp_host", - "scenario": "Use the LocalStack chaos injector to add 100 milliseconds of network latency to LocalStack, then read back the currently configured latency to verify it was applied, and finally clear the latency so LocalStack returns to normal.", + "scenario": "Simulate a slow network: make every request to my local AWS emulator 100 milliseconds slower, check that the delay is in effect, then remove it again.", "mcpHostConfig": { "provider": "openrouter", "model": "google/gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, "expect": { "toolsTriggered": { "calls": [ @@ -280,110 +176,49 @@ } }, { - "id": "scenario-extensions-marketplace", - "description": "localstack-extensions: browse the marketplace catalog.", - "mode": "mcp_host", - "scenario": "Browse the LocalStack Extensions marketplace and show me which extensions are available to install. I want the marketplace catalog of available extensions, not the list of extensions that are already installed.", - "mcpHostConfig": { - "provider": "openrouter", - "model": "google/gemini-2.5-flash", - "temperature": 0 - }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-extensions", - "arguments": { "action": "available" }, - "required": true - } - ], - "order": "any" - }, - "matchesPattern": [ - "\"output\":\"(?:[^\"\\\\]|\\\\.)*\\d+ extensions available\\. Install any" - ] - } - }, - { - "id": "scenario-replicator-read-only", - "description": "localstack-aws-replicator: read-only supported-resources and job listing.", + "id": "scenario-app-inspector-traces", + "description": "localstack-app-inspector: list traces, enabling App Inspector first if needed.", "mode": "mcp_host", - "scenario": "Using the LocalStack AWS replicator tool, first show me which resource types the AWS replicator supports for replication into LocalStack, and then list any replication jobs that currently exist. Do not start any replication job.", + "scenario": "Show me the most recent traces of the AWS API calls made against my local emulator. If tracing is turned off, turn it on first.", "mcpHostConfig": { "provider": "openrouter", "model": "google/gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, "expect": { "toolsTriggered": { "calls": [ { - "name": "localstack-aws-replicator", - "arguments": { "action": "list-resources" }, - "required": true - }, - { - "name": "localstack-aws-replicator", - "arguments": { "action": "list" }, + "name": "localstack-app-inspector", + "arguments": { "action": "list-traces" }, "required": true } ], "order": "any" }, - "matchesPattern": [ - "\"output\":\"(?:[^\"\\\\]|\\\\.)*## AWS Replicator Supported Resources", - "\"output\":\"(?:[^\"\\\\]|\\\\.)*## AWS Replicator Jobs" - ] + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*## Traces"] } }, { - "id": "scenario-ephemeral-list", - "description": "localstack-ephemeral-instances: read-only listing.", + "id": "scenario-docs-search", + "description": "localstack-docs: a how-it-works question the docs answer, next to IAM tools that act on the running instance.", "mode": "mcp_host", - "scenario": "List all LocalStack Ephemeral Instances that currently exist in my LocalStack Cloud account using the ephemeral instances tool, and summarize their names and statuses. Do not create, modify, or delete any instance.", + "scenario": "How does IAM enforcement work in LocalStack, and how do its enforcement modes differ? Look it up in the documentation instead of answering from memory.", "mcpHostConfig": { "provider": "openrouter", "model": "google/gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, "expect": { "toolsTriggered": { "calls": [ { - "name": "localstack-ephemeral-instances", - "arguments": { "action": "list" }, + "name": "localstack-docs", "required": true } ], "order": "any" }, - "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*## Ephemeral Instances"] - } - }, - { - "id": "scenario-docs-search", - "description": "localstack-docs: documentation search.", - "mode": "mcp_host", - "scenario": "Search the official LocalStack documentation for how IAM policy enforcement modes work in LocalStack and summarize what the docs say. Use the LocalStack documentation search tool, not the IAM tools that act on the running instance.", - "mcpHostConfig": { - "provider": "openrouter", - "model": "google/gemini-2.5-flash", - "temperature": 0 - }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [{ "name": "localstack-docs", "required": true }], - "order": "any" - }, "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*\\*\\*[1-9]\\d* results? found\\*\\*"] } }, @@ -398,8 +233,6 @@ "temperature": 0, "maxToolCalls": 15 }, - "iterations": 3, - "accuracyThreshold": 0.66, "expect": { "toolsTriggered": { "calls": [ diff --git a/package.json b/package.json index c8d202b..826d793 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,8 @@ "test": "jest", "test:mcp:direct": "yarn build && playwright test -c playwright.config.mjs tests/mcp/direct.spec.mjs tests/mcp/azure-offline.spec.mjs", "test:mcp:evals": "yarn build && playwright test -c playwright.config.mjs tests/mcp/evals.spec.mjs", - "test:mcp": "yarn test:mcp:direct && yarn test:mcp:evals" + "test:mcp:tools": "yarn build && playwright test -c playwright.config.mjs tests/mcp/tools.spec.mjs", + "test:mcp": "yarn test:mcp:direct && yarn test:mcp:tools && yarn test:mcp:evals" }, "dependencies": { "ajv": "^8.18.0", diff --git a/tests/mcp/tools.spec.mjs b/tests/mcp/tools.spec.mjs new file mode 100644 index 0000000..b4d4dc7 --- /dev/null +++ b/tests/mcp/tools.spec.mjs @@ -0,0 +1,145 @@ +import { expect, test } from "@gleanwork/mcp-server-tester/fixtures/mcp"; + +// One deterministic call sequence per tool against a running LocalStack: no model involved, so +// these run on every pull request. The evals (evals.spec.mjs) cover whether a model picks and +// combines the tools; these cover whether each tool works. +// +// The tests share one LocalStack and run in order: the first starts LocalStack if needed, the +// deploy creates the bucket that later tests look for. To use an instance other than the +// default on 4566, set LOCALSTACK_PORT and MAIN_CONTAINER_NAME. + +// The tester's callTool applies the MCP SDK's 60s request timeout; a start, a deploy or a Cloud +// Pod save can take longer, so call the SDK client directly. +const CALL_TIMEOUT_MS = 300_000; +const BUCKET = "mcpjam-eval-bucket"; // data/sample-terraform's default name_prefix +const POD = `mcp-tools-test-${process.env.GITHUB_RUN_ID ?? Date.now()}`; + +async function callRaw(mcp, name, args) { + const result = await mcp.client.callTool({ name, arguments: args }, undefined, { + timeout: CALL_TIMEOUT_MS, + }); + const text = (result.content ?? []).map((block) => block.text ?? "").join("\n"); + return { isError: result.isError === true, text }; +} + +// Returns the tool's text, failing with that text when the tool reports an error. +async function call(mcp, name, args) { + const { isError, text } = await callRaw(mcp, name, args); + expect(isError, `${name} ${JSON.stringify(args)} failed:\n${text}`).toBe(false); + return text; +} + +test.describe("tools against a running LocalStack", () => { + test.describe.configure({ mode: "serial", timeout: 600_000 }); + + test.beforeAll(() => { + if (!process.env.LOCALSTACK_AUTH_TOKEN?.trim()) { + throw new Error("Missing required environment variable: LOCALSTACK_AUTH_TOKEN"); + } + }); + + test("management: starts LocalStack and reports it ready", async ({ mcp }) => { + const before = await callRaw(mcp, "localstack-management", { action: "status" }); + if (!/Services initialized/.test(before.text)) { + await call(mcp, "localstack-management", { action: "start" }); + } + const status = await call(mcp, "localstack-management", { action: "status" }); + expect(status).toMatch(/Services initialized: [1-9]\d*\/\d+/); + }); + + test("app-inspector: enables tracing", async ({ mcp }) => { + await call(mcp, "localstack-app-inspector", { action: "set-status", status: "enabled" }); + const status = await call(mcp, "localstack-app-inspector", { action: "get-status" }); + expect(status).toContain("## App Inspector Status"); + expect(status).toMatch(/enabled/i); + }); + + test("deployer: applies the sample Terraform project", async ({ mcp }) => { + const report = await call(mcp, "localstack-deployer", { + action: "deploy", + projectType: "terraform", + directory: "data/sample-terraform", + }); + expect(report).toContain("Apply complete!"); + expect(report).toContain(BUCKET); + }); + + test("aws-client: lists the deployed bucket", async ({ mcp }) => { + expect(await call(mcp, "localstack-aws-client", { command: "s3 ls" })).toContain(BUCKET); + }); + + test("state-management: shows S3 state", async ({ mcp }) => { + const state = await call(mcp, "localstack-state-management", { action: "inspect" }); + expect(state).toContain("## LocalStack State Inspect"); + expect(state).toMatch(/\bs3\b/i); + }); + + test("app-inspector: lists traces of the deploy", async ({ mcp }) => { + const traces = await call(mcp, "localstack-app-inspector", { action: "list-traces" }); + expect(traces).toMatch(/## Traces \([1-9]\d*\)/); + }); + + test("logs-analysis: runs an errors analysis", async ({ mcp }) => { + const logs = await call(mcp, "localstack-logs-analysis", { analysisType: "errors" }); + expect(logs).toMatch(/# 🚨 LocalStack Errors|βœ… No errors found/); + }); + + test("iam-policy-analyzer: switches to soft mode", async ({ mcp }) => { + expect(await call(mcp, "localstack-iam-policy-analyzer", { action: "get-status" })).toContain( + "**LocalStack IAM Enforcement Status**" + ); + await call(mcp, "localstack-iam-policy-analyzer", { action: "set-mode", mode: "SOFT_MODE" }); + expect(await call(mcp, "localstack-iam-policy-analyzer", { action: "get-status" })).toContain( + "SOFT_MODE" + ); + }); + + test("chaos-injector: injects, reads and clears latency", async ({ mcp }) => { + const tool = "localstack-chaos-injector"; + expect(await call(mcp, tool, { action: "inject-latency", latency_ms: 100 })).toContain( + "Latency of 100ms has been injected" + ); + expect(await call(mcp, tool, { action: "get-latency" })).toContain( + "The current network latency is 100ms" + ); + expect(await call(mcp, tool, { action: "clear-latency" })).toContain( + "Network latency has been cleared" + ); + }); + + test("cloud-pods: saves, loads and deletes a pod", async ({ mcp }) => { + const tool = "localstack-cloud-pods"; + try { + expect(await call(mcp, tool, { action: "save", pod_name: POD })).toContain( + "was saved successfully" + ); + expect(await call(mcp, tool, { action: "load", pod_name: POD })).toContain( + "Your LocalStack instance has been restored" + ); + } finally { + expect(await call(mcp, tool, { action: "delete", pod_name: POD })).toContain( + "has been permanently deleted" + ); + } + }); + + test("aws-replicator: lists resource types and jobs", async ({ mcp }) => { + const tool = "localstack-aws-replicator"; + expect(await call(mcp, tool, { action: "list-resources" })).toContain( + "## AWS Replicator Supported Resources" + ); + expect(await call(mcp, tool, { action: "list" })).toContain("## AWS Replicator Jobs"); + }); + + test("extensions: lists the marketplace", async ({ mcp }) => { + expect(await call(mcp, "localstack-extensions", { action: "available" })).toMatch( + /[1-9]\d* extensions available\. Install any/ + ); + }); + + test("ephemeral-instances: lists instances", async ({ mcp }) => { + expect(await call(mcp, "localstack-ephemeral-instances", { action: "list" })).toContain( + "## Ephemeral Instances" + ); + }); +}); From e8a0ceb763044d6f838eedfd98cfd718a2a57a50 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Thu, 19 Nov 2026 10:45:45 +0100 Subject: [PATCH 05/13] test: require the host-tools action in jobs that run the tool tests or evals The pin test only banned the setup actions and download URLs, so a job could install Terraform or uv with pip, pipx, brew, apt, mise or asdf, or skip the action and use whatever the runner image ships. It now bans those installs too, and fails when a job that runs `test:mcp`, `test:mcp:tools` or `test:mcp:evals` doesn't use .github/actions/host-tools. Co-Authored-By: Claude Opus 5.5 --- .github/actions/host-tools/action.yml | 2 +- Dockerfile | 1 + src/core/dependency-pins.test.ts | 18 +++++++++++++++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/actions/host-tools/action.yml b/.github/actions/host-tools/action.yml index 83a466d..ad4244c 100644 --- a/.github/actions/host-tools/action.yml +++ b/.github/actions/host-tools/action.yml @@ -1,7 +1,7 @@ name: Host tools description: >- Puts the Docker image's tools on the PATH for jobs that run the MCP server on the host: - Terraform and uv from the Dockerfile's host-tools stage, lstk from docker/tools/npm, and + always Terraform and uv (from the Dockerfile's host-tools stage) and lstk (docker/tools/npm), and optionally the Snowflake CLI from docker/tools/snowflake. Versions change only in those files. inputs: snowflake: diff --git a/Dockerfile b/Dockerfile index 8f37794..ac5c2d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ # aws-sam-cli docker/tools/sam/uv.lock (uv) # snowflake-cli docker/tools/snowflake/uv.lock (uv) # dockerode & co. yarn.lock (npm, same lock as CI) +# The host-tools stage hands the same terraform and uv binaries to CI jobs that run on the host. # Debian packages (python3, git, curl) come with the node image's bookworm base. # Both node FROM lines must stay identical, and their major must match `node-version` in the # workflows; src/core/dependency-pins.test.ts checks this. diff --git a/src/core/dependency-pins.test.ts b/src/core/dependency-pins.test.ts index ce9c627..220a4b7 100644 --- a/src/core/dependency-pins.test.ts +++ b/src/core/dependency-pins.test.ts @@ -109,10 +109,22 @@ describe("dependency pins", () => { ...fs.readdirSync(path.join(ROOT, ".github/workflows")).map((f) => `.github/workflows/${f}`), ".github/actions/host-tools/action.yml", ]; + const otherInstall = + /setup-terraform|setup-uv|terraform_version|releases\.hashicorp\.com|astral\.sh\/uv|\b(?:pipx?|brew|apt(?:-get)?|mise|asdf) install\b.*\b(?:terraform|uv)\b/g; for (const file of files) { - expect( - `${file}: ${read(file).match(/setup-terraform|setup-uv|terraform_version|releases\.hashicorp\.com|astral\.sh\/uv/g) ?? []}` - ).toBe(`${file}: `); + expect(`${file}: ${read(file).match(otherInstall) ?? []}`).toBe(`${file}: `); + } + + // Jobs that run the tool tests or the evals need the tools; without the action they would + // use whatever the runner image ships. Jobs are split at their two-space-indented keys. + for (const file of fs.readdirSync(path.join(ROOT, ".github/workflows"))) { + const jobs = read(`.github/workflows/${file}`).split(/^jobs:\s*$/m)[1] ?? ""; + for (const job of jobs.split(/^(?= [\w-]+:\s*$)/m)) { + if (!/yarn test:mcp(?::tools|:evals)?\s*$/m.test(job)) continue; + expect( + `${file} ${job.trim().split(":")[0]}: ${/uses: \.\/\.github\/actions\/host-tools\b/.test(job)}` + ).toBe(`${file} ${job.trim().split(":")[0]}: true`); + } } }); From a91849cb21fa5c88200514ea3e09a9ce8e3a9793 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Thu, 19 Nov 2026 11:36:20 +0100 Subject: [PATCH 06/13] test: record eval tokens and cost by patching mcp-server-tester 1.1.1 The tester's AI SDK host read `usage.promptTokens` and `completionTokens`, names the AI SDK dropped in v5, and hardcoded `totalCostUsd: 0`, so every report showed 0 tokens and $0. A patch-package patch reads `totalUsage`, which covers every step of a multi-step case, and sums the cost OpenRouter returns for each step. `prepare` applies it on install; unlike `postinstall` it doesn't run when users install the published package. When Dependabot bumps the tester, the patch must be redone or dropped. 2.0 counts tokens but not cost. Co-Authored-By: Claude Opus 5.5 --- package.json | 2 + .../@gleanwork+mcp-server-tester+1.1.1.patch | 36 +++ yarn.lock | 217 +++++++++++++++++- 3 files changed, 249 insertions(+), 6 deletions(-) create mode 100644 patches/@gleanwork+mcp-server-tester+1.1.1.patch diff --git a/package.json b/package.json index 826d793..b8775dd 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "dev": "xmcp dev", "start": "node dist/cli.js", "prepack": "yarn build", + "prepare": "patch-package", "format": "prettier --write .", "test": "jest", "test:mcp:direct": "yarn build && playwright test -c playwright.config.mjs tests/mcp/direct.spec.mjs tests/mcp/azure-offline.spec.mjs", @@ -37,6 +38,7 @@ "eslint-config-prettier": "^10.1.8", "jest": "^30.5.2", "jsonc-parser": "^3.3.1", + "patch-package": "^8.0.1", "posthog-node": "5.73.1", "prettier": "^3.9.10", "swc-loader": "^0.2.6", diff --git a/patches/@gleanwork+mcp-server-tester+1.1.1.patch b/patches/@gleanwork+mcp-server-tester+1.1.1.patch new file mode 100644 index 0000000..03da55b --- /dev/null +++ b/patches/@gleanwork+mcp-server-tester+1.1.1.patch @@ -0,0 +1,36 @@ +diff --git a/node_modules/@gleanwork/mcp-server-tester/dist/index.cjs b/node_modules/@gleanwork/mcp-server-tester/dist/index.cjs +index 1361291..4c0d0e2 100644 +--- a/node_modules/@gleanwork/mcp-server-tester/dist/index.cjs ++++ b/node_modules/@gleanwork/mcp-server-tester/dist/index.cjs +@@ -6965,9 +6965,10 @@ function createVercelOrchestrator() { + const totalDurationMs = Date.now() - llmStart; + const llmDurationMs = totalDurationMs - mcpDurationMs; + const hostUsage = result.usage ? { +- inputTokens: result.usage.promptTokens ?? 0, +- outputTokens: result.usage.completionTokens ?? 0, +- totalCostUsd: 0, ++ // AI SDK 5+ names: totalUsage covers every step; OpenRouter reports each step's cost. ++ inputTokens: (result.totalUsage ?? result.usage).inputTokens ?? 0, ++ outputTokens: (result.totalUsage ?? result.usage).outputTokens ?? 0, ++ totalCostUsd: (result.steps ?? []).reduce((sum, step) => sum + (step.providerMetadata?.openrouter?.usage?.cost ?? 0), 0), + durationMs: llmDurationMs + } : void 0; + const conversationHistory = (result.steps ?? []).flatMap((step) => { +diff --git a/node_modules/@gleanwork/mcp-server-tester/dist/index.js b/node_modules/@gleanwork/mcp-server-tester/dist/index.js +index 71c1ef0..8c444e9 100644 +--- a/node_modules/@gleanwork/mcp-server-tester/dist/index.js ++++ b/node_modules/@gleanwork/mcp-server-tester/dist/index.js +@@ -6938,9 +6938,10 @@ function createVercelOrchestrator() { + const totalDurationMs = Date.now() - llmStart; + const llmDurationMs = totalDurationMs - mcpDurationMs; + const hostUsage = result.usage ? { +- inputTokens: result.usage.promptTokens ?? 0, +- outputTokens: result.usage.completionTokens ?? 0, +- totalCostUsd: 0, ++ // AI SDK 5+ names: totalUsage covers every step; OpenRouter reports each step's cost. ++ inputTokens: (result.totalUsage ?? result.usage).inputTokens ?? 0, ++ outputTokens: (result.totalUsage ?? result.usage).outputTokens ?? 0, ++ totalCostUsd: (result.steps ?? []).reduce((sum, step) => sum + (step.providerMetadata?.openrouter?.usage?.cost ?? 0), 0), + durationMs: llmDurationMs + } : void 0; + const conversationHistory = (result.steps ?? []).flatMap((step) => { diff --git a/yarn.lock b/yarn.lock index 443a457..e240cef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1890,6 +1890,11 @@ ts-checker-rspack-plugin "^1.8.0" typescript "^5.9.3" +"@yarnpkg/lockfile@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz#e77a97fbd345b76d83245edcd17d393b1b41fb31" + integrity sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ== + abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" @@ -2184,6 +2189,13 @@ brace-expansion@^5.0.8: dependencies: balanced-match "^4.0.2" +braces@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== + dependencies: + fill-range "^7.1.1" + browserslist@^4.24.0: version "4.28.7" resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.28.7.tgz#409046517fccd2e51cdc20f077454b7141184028" @@ -2247,7 +2259,17 @@ call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2: es-errors "^1.3.0" function-bind "^1.1.2" -call-bound@^1.0.2: +call-bind@^1.0.8: + version "1.0.9" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.9.tgz#39a644700c80bc7d0ca9102fc6d1d43b2fd7eee7" + integrity sha512-a/hy+pNsFUTR+Iz8TCJvXudKVLAnz/DyeSUo10I5yvFDQJBFU2s9uqQpoSrJlroHUKoKqzg+epxyP9lqFdzfBQ== + dependencies: + call-bind-apply-helpers "^1.0.2" + es-define-property "^1.0.1" + get-intrinsic "^1.3.0" + set-function-length "^1.2.2" + +call-bound@^1.0.2, call-bound@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/call-bound/-/call-bound-1.0.4.tgz#238de935d2a2a692928c538c7ccfa91067fd062a" integrity sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg== @@ -2305,6 +2327,11 @@ chownr@^1.1.1: resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== +ci-info@^3.7.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.9.0.tgz#4279a62028a7b1f262f3473fc9605f5e218c59b4" + integrity sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ== + ci-info@^4.2.0: version "4.4.0" resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-4.4.0.tgz#7d54eff9f54b45b62401c26032696eb59c8bd18c" @@ -2500,6 +2527,15 @@ default-browser@^5.2.1: bundle-name "^4.1.0" default-browser-id "^5.0.0" +define-data-property@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.4.tgz#894dc141bb7d3060ae4366f6a0107e68fbe48c5e" + integrity sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A== + dependencies: + es-define-property "^1.0.0" + es-errors "^1.3.0" + gopd "^1.0.1" + define-lazy-prop@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz#dbb19adfb746d7fc6d734a06b72f4a00d021255f" @@ -2642,7 +2678,7 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -es-define-property@^1.0.1: +es-define-property@^1.0.0, es-define-property@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.1.tgz#983eb2f9a6724e9303f61addf011c72e09e0b0fa" integrity sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g== @@ -2948,6 +2984,13 @@ figures@^6.1.0: dependencies: is-unicode-supported "^2.0.0" +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== + dependencies: + to-regex-range "^5.0.1" + finalhandler@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-2.1.1.tgz#a2c517a6559852bcdb06d1f8bd7f51b68fad8099" @@ -2968,6 +3011,13 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" +find-yarn-workspace-root@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/find-yarn-workspace-root/-/find-yarn-workspace-root-2.0.0.tgz#f47fb8d239c900eb78179aa81b66673eac88f7bd" + integrity sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ== + dependencies: + micromatch "^4.0.2" + foreground-child@^3.1.0: version "3.3.1" resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.3.1.tgz#32e8e9ed1b68a3497befb9ac2b6adf92a638576f" @@ -3015,6 +3065,15 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== +fs-extra@^10.0.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" + integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + function-bind@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" @@ -3073,7 +3132,7 @@ get-east-asian-width@^1.0.0, get-east-asian-width@^1.3.1: resolved "https://registry.yarnpkg.com/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz#216900f91df11a8b2c198c3e1d93d6c035a776b9" integrity sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA== -get-intrinsic@^1.2.5, get-intrinsic@^1.2.6, get-intrinsic@^1.3.0: +get-intrinsic@^1.2.4, get-intrinsic@^1.2.5, get-intrinsic@^1.2.6, get-intrinsic@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01" integrity sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ== @@ -3172,11 +3231,16 @@ google-logging-utils@^1.0.0: resolved "https://registry.yarnpkg.com/google-logging-utils/-/google-logging-utils-1.2.0.tgz#89dd643f986c765e61bfa2d01cada2292fe5bfd7" integrity sha512-WE9av4wKDZgRjBwgVUabocx8T6/7o3Ca1Fat46FXDhXVAFibzNadedcOXrdgd1Kzmk8tsk/9ZH89Wyf/SqeZ3A== -gopd@^1.2.0: +gopd@^1.0.1, gopd@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1" integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg== +graceful-fs@^4.1.11, graceful-fs@^4.1.6, graceful-fs@^4.2.0: + version "4.2.12" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.12.tgz#7f6162dee07bc600281cea1b038e5f7cf215a6c2" + integrity sha512-iuyGF2E4oafQfPsRrG2wtY2ILupMhvtdrhzrxB+0VcjsPpSPttpfYdjnGFGsTi5MaJFjI1jOKHPQ6UTbTj9TAw== + graceful-fs@^4.2.11: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" @@ -3207,6 +3271,13 @@ has-flag@^4.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== +has-property-descriptors@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz#963ed7d071dc7bf5f084c5bfbe0d1b6222586854" + integrity sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg== + dependencies: + es-define-property "^1.0.0" + has-symbols@^1.0.3, has-symbols@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.1.0.tgz#fc9c6a783a084951d0b971fe1018de813707a338" @@ -3382,6 +3453,11 @@ is-arrayish@^0.2.1: resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" integrity sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg== +is-docker@^2.0.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.2.1.tgz#33eeabe23cfe86f14bde4408a02c0cfb853acdaa" + integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ== + is-docker@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-3.0.0.tgz#90093aa3106277d8a77a5910dbae71747e15a200" @@ -3433,6 +3509,11 @@ is-inside-container@^1.0.0: dependencies: is-docker "^3.0.0" +is-number@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" + integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== + is-promise@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-4.0.0.tgz#42ff9f84206c1991d26debf520dd5c01042dd2f3" @@ -3453,6 +3534,13 @@ is-unsafe@^2.0.3: resolved "https://registry.yarnpkg.com/is-unsafe/-/is-unsafe-2.0.3.tgz#c8bc72c9d354dc9483dab9c26ff69bf5ec68d7b0" integrity sha512-ZBllczCCO267vgdFb6Tq3pv+LhHaCiGMdr+ZsuL+OopruAg7S+3ZDs1vD0RPC0uSrkjzbraxaGjtX1YBxjPeFQ== +is-wsl@^2.1.1: + version "2.2.0" + resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-2.2.0.tgz#74a4c76e77ca9fd3f932f290c17ea326cd157271" + integrity sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww== + dependencies: + is-docker "^2.0.0" + is-wsl@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-3.1.1.tgz#327897b26832a3eb117da6c27492d04ca132594f" @@ -3460,6 +3548,11 @@ is-wsl@^3.1.0: dependencies: is-inside-container "^1.0.0" +isarray@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723" + integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== + isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" @@ -4005,6 +4098,17 @@ json-schema@^0.4.0: resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== +json-stable-stringify@^1.0.2: + version "1.3.0" + resolved "https://registry.yarnpkg.com/json-stable-stringify/-/json-stable-stringify-1.3.0.tgz#8903cfac42ea1a0f97f35d63a4ce0518f0cc6a70" + integrity sha512-qtYiSSFlwot9XHtF9bD9c7rwKjr+RecWT//ZnPvSmEjpV5mmPOCN4j8UjY5hbjNkOwZ/jQv3J6R1/pL7RwgMsg== + dependencies: + call-bind "^1.0.8" + call-bound "^1.0.4" + isarray "^2.0.5" + jsonify "^0.0.1" + object-keys "^1.1.1" + json5@^2.2.3: version "2.2.3" resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" @@ -4015,6 +4119,20 @@ jsonc-parser@^3.3.1: resolved "https://registry.yarnpkg.com/jsonc-parser/-/jsonc-parser-3.3.1.tgz#f2a524b4f7fd11e3d791e559977ad60b98b798b4" integrity sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ== +jsonfile@^6.0.1: + version "6.2.1" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.2.1.tgz#b6e31717f22cc37330b081ce0051ed5de53af2f6" + integrity sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q== + dependencies: + universalify "^2.0.0" + optionalDependencies: + graceful-fs "^4.1.6" + +jsonify@^0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.1.tgz#2aa3111dae3d34a0f151c63f3a45d995d9420978" + integrity sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg== + jsonparse@^1.2.0: version "1.3.1" resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" @@ -4045,6 +4163,13 @@ jws@^4.0.0: jwa "^2.0.1" safe-buffer "^5.0.1" +klaw-sync@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/klaw-sync/-/klaw-sync-6.0.0.tgz#1fd2cfd56ebb6250181114f0a581167099c2b28c" + integrity sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ== + dependencies: + graceful-fs "^4.1.11" + leven@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/leven/-/leven-3.1.0.tgz#77891de834064cccba82ae7842bb6b14a13ed7f2" @@ -4158,6 +4283,14 @@ merge-stream@^2.0.0: resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== +micromatch@^4.0.2: + version "4.0.8" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" + integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== + dependencies: + braces "^3.0.3" + picomatch "^2.3.1" + mime-db@1.52.0: version "1.52.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" @@ -4206,7 +4339,7 @@ minimatch@^9.0.4: dependencies: brace-expansion "^2.0.2" -minimist@^1.2.8: +minimist@^1.2.6, minimist@^1.2.8: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== @@ -4314,6 +4447,11 @@ object-inspect@^1.13.3, object-inspect@^1.13.4: resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.4.tgz#8375265e21bc20d0fa582c22e1b13485d6e00213" integrity sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew== +object-keys@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" + integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== + on-finished@^2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" @@ -4345,6 +4483,14 @@ open@^10.1.0: is-inside-container "^1.0.0" wsl-utils "^0.1.0" +open@^7.4.2: + version "7.4.2" + resolved "https://registry.yarnpkg.com/open/-/open-7.4.2.tgz#b8147e26dcf3e426316c730089fd71edd29c2321" + integrity sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q== + dependencies: + is-docker "^2.0.0" + is-wsl "^2.1.1" + p-limit@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" @@ -4403,6 +4549,26 @@ patch-console@^2.0.0: resolved "https://registry.yarnpkg.com/patch-console/-/patch-console-2.0.0.tgz#9023f4665840e66f40e9ce774f904a63167433bb" integrity sha512-0YNdUceMdaQwoKce1gatDScmMo5pu/tfABfnzEqeG0gtTmd7mh/WcwgUjtAeOU7N8nFFlbQBnFK2gXW5fGvmMA== +patch-package@^8.0.1: + version "8.0.1" + resolved "https://registry.yarnpkg.com/patch-package/-/patch-package-8.0.1.tgz#79d02f953f711e06d1f8949c8a13e5d3d7ba1a60" + integrity sha512-VsKRIA8f5uqHQ7NGhwIna6Bx6D9s/1iXlA1hthBVBEbkq+t4kXD0HHt+rJhf/Z+Ci0F/HCB2hvn0qLdLG+Qxlw== + dependencies: + "@yarnpkg/lockfile" "^1.1.0" + chalk "^4.1.2" + ci-info "^3.7.0" + cross-spawn "^7.0.3" + find-yarn-workspace-root "^2.0.0" + fs-extra "^10.0.0" + json-stable-stringify "^1.0.2" + klaw-sync "^6.0.0" + minimist "^1.2.6" + open "^7.4.2" + semver "^7.5.3" + slash "^2.0.0" + tmp "^0.2.4" + yaml "^2.2.2" + path-exists@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" @@ -4451,7 +4617,7 @@ picocolors@^1.1.1: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== -picomatch@^2.0.4: +picomatch@^2.0.4, picomatch@^2.3.1: version "2.3.2" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.2.tgz#5a942915e26b372dc0f0e6753149a16e6b1c5601" integrity sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA== @@ -4760,6 +4926,18 @@ serve-static@^2.2.0: parseurl "^1.3.3" send "^1.2.0" +set-function-length@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449" + integrity sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg== + dependencies: + define-data-property "^1.1.4" + es-errors "^1.3.0" + function-bind "^1.1.2" + get-intrinsic "^1.2.4" + gopd "^1.0.1" + has-property-descriptors "^1.0.2" + setprototypeof@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" @@ -4832,6 +5010,11 @@ sisteransi@^2.0.0: resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-2.0.0.tgz#d3d7a694075c2b17094cf1a691afd7937643b936" integrity sha512-2WCqLdt12eN6ehLI50S2/ZxoowXU3DD1KVPRyVP0WEsFahrGZZmCdq1K59qjJljMRIDTwx1D9YaRK+hDneBUMg== +slash@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/slash/-/slash-2.0.0.tgz#de552851a1759df3a8f206535442f5ec4ddeab44" + integrity sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A== + slash@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" @@ -5103,6 +5286,18 @@ through@2, through@^2.3.8, through@~2.3, through@~2.3.4: resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== +tmp@^0.2.4: + version "0.2.7" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.7.tgz#26f4db11d1601ce8012dcb8a798ece1c06a99059" + integrity sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw== + +to-regex-range@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" + integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ== + dependencies: + is-number "^7.0.0" + toidentifier@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" @@ -5212,6 +5407,11 @@ undici@^7.24.0: resolved "https://registry.yarnpkg.com/undici/-/undici-7.30.1.tgz#360b3de104b94abc52729aa1a4a07e027aa71f73" integrity sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg== +universalify@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.1.tgz#168efc2180964e6386d061e094df61afe239b18d" + integrity sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw== + unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" @@ -5397,6 +5597,11 @@ yallist@^3.0.2: resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== +yaml@^2.2.2: + version "2.9.1" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.9.1.tgz#c16233fb31944e705cfefaff38795587f57588ce" + integrity sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw== + yargs-parser@^21.1.1: version "21.1.1" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35" From a9402c4fd7d824b5a6f2a6bfaded95cf874aad0e Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Thu, 19 Nov 2026 12:09:30 +0100 Subject: [PATCH 07/13] test: check the state tools leave behind, and fix review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mutation testing showed the tool tests passed when a tool skipped its API call but still reported success, because they checked the tool's own message. Each test now reads the state back: IAM and App Inspector switch between two states and read each back, chaos reads the latency after clearing it, Cloud Pods removes the bucket after the save and checks the load brings it back, a second delete must fail, traces must list a row, the logs analysis must report a request made to fail, the deploy passes name_prefix and checks the outputs table, and the region-bound SQS queue must be listed. The evals fail a case when every call of a required tool returned an error (the tester drops isError; this server's errors start with ❌), and the chaos and IAM prompts ask for a read-back that the patterns check. Review fixes: fork PRs skip the jobs that need secrets; the Cloud Pods test cleans up without masking the first error and names the pod per attempt; chaos and IAM restore state on failure; Dependabot no longer bumps the patched tester; the pin test also catches `yarn run`/`npm run` jobs and curl installers, and no longer flags links to the uv docs. Co-Authored-By: Claude Opus 5.5 --- .github/dependabot.yml | 3 + .github/workflows/ci.yml | 4 ++ data/evals/comprehensive.json | 10 +-- src/core/dependency-pins.test.ts | 4 +- tests/mcp/evals.spec.mjs | 26 +++++++- tests/mcp/tools.spec.mjs | 110 +++++++++++++++++++++---------- 6 files changed, 116 insertions(+), 41 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5a733f1..eab09f9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,6 +21,9 @@ updates: - dependency-name: zod update-types: - version-update:semver-minor + # patches/ holds a patch for this exact version, and patch-package fails the install in CI + # when it no longer applies. Bump it by hand and redo or drop the patch. + - dependency-name: "@gleanwork/mcp-server-tester" # CLIs inside the Docker image (aws-cdk, lstk) and the CDK sample app that CI deploys with them. - package-ecosystem: npm diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 20c159c..8417833 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,8 @@ jobs: run: yarn build mcp-direct-tests: + # Fork PRs get no secrets, so they skip it; branches in this repo and Dependabot run it. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-26.04 env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} @@ -60,6 +62,8 @@ jobs: # Each tool against a real LocalStack, without a model (tests/mcp/tools.spec.mjs). The first # test starts LocalStack through the management tool. mcp-tool-tests: + # Fork PRs get no secrets, so they skip it; branches in this repo and Dependabot run it. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-26.04 env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} diff --git a/data/evals/comprehensive.json b/data/evals/comprehensive.json index 47cd2cd..5177669 100644 --- a/data/evals/comprehensive.json +++ b/data/evals/comprehensive.json @@ -115,7 +115,7 @@ "id": "scenario-iam-soft-mode", "description": "localstack-iam-policy-analyzer: switch IAM enforcement to soft mode from a symptom, not the mode's name.", "mode": "mcp_host", - "scenario": "My Lambda function keeps failing with AccessDenied errors in my local AWS emulator. Instead of blocking those calls, let them through and only log which permissions are missing.", + "scenario": "My Lambda function keeps failing with AccessDenied errors in my local AWS emulator. Instead of blocking those calls, let them through and only log which permissions are missing. Then check which mode is active.", "mcpHostConfig": { "provider": "openrouter", "model": "google/gemini-2.5-flash", @@ -133,7 +133,8 @@ "order": "any" }, "matchesPattern": [ - "\"output\":\"(?:[^\"\\\\]|\\\\.)*IAM enforcement mode has been set to `SOFT_MODE`" + "\"output\":\"(?:[^\"\\\\]|\\\\.)*IAM enforcement mode has been set to `SOFT_MODE`", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*\\*\\*Current Mode:\\*\\* `SOFT_MODE`" ] } }, @@ -141,7 +142,7 @@ "id": "scenario-chaos-latency-cycle", "description": "localstack-chaos-injector: inject, read back and clear network latency.", "mode": "mcp_host", - "scenario": "Simulate a slow network: make every request to my local AWS emulator 100 milliseconds slower, check that the delay is in effect, then remove it again.", + "scenario": "Simulate a slow network: make every request to my local AWS emulator 100 milliseconds slower, check that the delay is in effect, then remove it again and check that it is gone.", "mcpHostConfig": { "provider": "openrouter", "model": "google/gemini-2.5-flash", @@ -171,7 +172,8 @@ "matchesPattern": [ "\"output\":\"(?:[^\"\\\\]|\\\\.)*Latency of 100ms has been injected", "\"output\":\"(?:[^\"\\\\]|\\\\.)*The current network latency is 100ms", - "\"output\":\"(?:[^\"\\\\]|\\\\.)*Network latency has been cleared" + "\"output\":\"(?:[^\"\\\\]|\\\\.)*Network latency has been cleared", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*The current network latency is 0ms" ] } }, diff --git a/src/core/dependency-pins.test.ts b/src/core/dependency-pins.test.ts index 220a4b7..b279c97 100644 --- a/src/core/dependency-pins.test.ts +++ b/src/core/dependency-pins.test.ts @@ -110,7 +110,7 @@ describe("dependency pins", () => { ".github/actions/host-tools/action.yml", ]; const otherInstall = - /setup-terraform|setup-uv|terraform_version|releases\.hashicorp\.com|astral\.sh\/uv|\b(?:pipx?|brew|apt(?:-get)?|mise|asdf) install\b.*\b(?:terraform|uv)\b/g; + /setup-terraform|setup-uv|terraform_version|\b(?:curl|wget)\b.*\b(?:hashicorp\.com|astral\.sh|astral-sh)\b|\b(?:pipx?|brew|apt(?:-get)?|mise|asdf) install\b.*\b(?:terraform|uv)\b/g; for (const file of files) { expect(`${file}: ${read(file).match(otherInstall) ?? []}`).toBe(`${file}: `); } @@ -120,7 +120,7 @@ describe("dependency pins", () => { for (const file of fs.readdirSync(path.join(ROOT, ".github/workflows"))) { const jobs = read(`.github/workflows/${file}`).split(/^jobs:\s*$/m)[1] ?? ""; for (const job of jobs.split(/^(?= [\w-]+:\s*$)/m)) { - if (!/yarn test:mcp(?::tools|:evals)?\s*$/m.test(job)) continue; + if (!/\b(?:yarn|npm|pnpm)(?: run)? test:mcp(?::tools|:evals)?\b(?!:)/.test(job)) continue; expect( `${file} ${job.trim().split(":")[0]}: ${/uses: \.\/\.github\/actions\/host-tools\b/.test(job)}` ).toBe(`${file} ${job.trim().split(":")[0]}: true`); diff --git a/tests/mcp/evals.spec.mjs b/tests/mcp/evals.spec.mjs index 8f8316b..05ed49e 100644 --- a/tests/mcp/evals.spec.mjs +++ b/tests/mcp/evals.spec.mjs @@ -31,17 +31,39 @@ test.describe("OpenRouter comprehensive eval", () => { const thresholds = new Map( dataset.cases.map((c) => [c.id, Math.ceil((c.accuracyThreshold ?? 1) * (c.iterations ?? 1))]) ); + // The tester drops a tool result's isError and keeps only its text, which this server starts + // with ❌ on an error. A model can still read a usable answer out of an error's text, so each + // required call needs at least one matching call that didn't fail. + const cases = new Map(dataset.cases.map((c) => [c.id, c])); + const matches = (call, expected) => + call.name === expected.name && + Object.entries(expected.arguments ?? {}).every( + ([key, value]) => JSON.stringify(call.arguments?.[key]) === JSON.stringify(value) + ); + const failedCalls = (entry, iteration) => + (cases.get(entry.id)?.expect?.toolsTriggered?.calls ?? []) + .filter((expected) => expected.required) + .filter( + (expected) => + !(iteration.response?.toolCalls ?? []).some( + (call) => matches(call, expected) && !String(call.output ?? "").startsWith("❌") + ) + ) + .map((expected) => expected.name); + const passesIteration = (entry, iteration) => + iteration.pass === true && failedCalls(entry, iteration).length === 0; const passesCase = (entry) => entry?.pass === true && - (entry.iterationResults ?? [entry]).filter((r) => r.pass === true).length >= + (entry.iterationResults ?? [entry]).filter((r) => passesIteration(entry, r)).length >= thresholds.get(entry.id); const passed = caseResults.filter(passesCase).length; const failed = caseResults.filter((entry) => !passesCase(entry)); if (failed.length > 0) { for (const entry of failed) { + const errored = (entry.iterationResults ?? [entry]).map((r) => failedCalls(entry, r)); console.error( - `Eval case failed: ${entry.id} (infrastructure errors: ${entry.infrastructureErrorCount ?? 0})` + `Eval case failed: ${entry.id} (infrastructure errors: ${entry.infrastructureErrorCount ?? 0}; required calls that only returned errors: ${JSON.stringify(errored)})` ); console.error(JSON.stringify(entry, null, 2)); } diff --git a/tests/mcp/tools.spec.mjs b/tests/mcp/tools.spec.mjs index b4d4dc7..f1bf52a 100644 --- a/tests/mcp/tools.spec.mjs +++ b/tests/mcp/tools.spec.mjs @@ -4,15 +4,22 @@ import { expect, test } from "@gleanwork/mcp-server-tester/fixtures/mcp"; // these run on every pull request. The evals (evals.spec.mjs) cover whether a model picks and // combines the tools; these cover whether each tool works. // +// Each test checks the state a tool leaves behind, read back through another call, not only the +// tool's own success message: a tool that skips its API call but still reports success must fail. +// // The tests share one LocalStack and run in order: the first starts LocalStack if needed, the -// deploy creates the bucket that later tests look for. To use an instance other than the -// default on 4566, set LOCALSTACK_PORT and MAIN_CONTAINER_NAME. +// deploy creates the bucket and queue that later tests look for. To use an instance other than +// the default on 4566, set LOCALSTACK_PORT and MAIN_CONTAINER_NAME. // The tester's callTool applies the MCP SDK's 60s request timeout; a start, a deploy or a Cloud // Pod save can take longer, so call the SDK client directly. const CALL_TIMEOUT_MS = 300_000; -const BUCKET = "mcpjam-eval-bucket"; // data/sample-terraform's default name_prefix -const POD = `mcp-tools-test-${process.env.GITHUB_RUN_ID ?? Date.now()}`; +// A name_prefix other than data/sample-terraform's default, so the test sees the variable applied. +const PREFIX = "mcp-tools"; +const BUCKET = `${PREFIX}-bucket`; +const QUEUE = `${PREFIX}-queue`; +// Unique per run and attempt, so a re-run never meets a pod an aborted attempt left behind. +const POD = `mcp-tools-test-${process.env.GITHUB_RUN_ID ?? Date.now()}-${process.env.GITHUB_RUN_ATTEMPT ?? 1}`; async function callRaw(mcp, name, args) { const result = await mcp.client.callTool({ name, arguments: args }, undefined, { @@ -47,25 +54,32 @@ test.describe("tools against a running LocalStack", () => { expect(status).toMatch(/Services initialized: [1-9]\d*\/\d+/); }); - test("app-inspector: enables tracing", async ({ mcp }) => { - await call(mcp, "localstack-app-inspector", { action: "set-status", status: "enabled" }); - const status = await call(mcp, "localstack-app-inspector", { action: "get-status" }); - expect(status).toContain("## App Inspector Status"); - expect(status).toMatch(/enabled/i); + // Disabling first means the enable has to change something, whatever an earlier run left. + test("app-inspector: disables and enables tracing", async ({ mcp }) => { + const tool = "localstack-app-inspector"; + await call(mcp, tool, { action: "set-status", status: "disabled" }); + expect(await call(mcp, tool, { action: "get-status" })).toMatch(/\*\*Status:\*\* disabled\b/i); + await call(mcp, tool, { action: "set-status", status: "enabled" }); + expect(await call(mcp, tool, { action: "get-status" })).toMatch(/\*\*Status:\*\* enabled\b/i); }); - test("deployer: applies the sample Terraform project", async ({ mcp }) => { + test("deployer: applies the sample Terraform project with a variable", async ({ mcp }) => { const report = await call(mcp, "localstack-deployer", { action: "deploy", projectType: "terraform", directory: "data/sample-terraform", + variables: { name_prefix: PREFIX }, }); expect(report).toContain("Apply complete!"); - expect(report).toContain(BUCKET); + // The outputs table, not the apply log, which also prints the outputs. + expect(report).toContain(`| **bucket_name** | \`${BUCKET}\` |`); }); - test("aws-client: lists the deployed bucket", async ({ mcp }) => { - expect(await call(mcp, "localstack-aws-client", { command: "s3 ls" })).toContain(BUCKET); + test("aws-client: lists the deployed bucket and queue", async ({ mcp }) => { + const tool = "localstack-aws-client"; + expect(await call(mcp, tool, { command: "s3 ls" })).toContain(BUCKET); + // SQS is regional, so this also catches a client that talks to the wrong region. + expect(await call(mcp, tool, { command: "sqs list-queues" })).toContain(QUEUE); }); test("state-management: shows S3 state", async ({ mcp }) => { @@ -77,49 +91,79 @@ test.describe("tools against a running LocalStack", () => { test("app-inspector: lists traces of the deploy", async ({ mcp }) => { const traces = await call(mcp, "localstack-app-inspector", { action: "list-traces" }); expect(traces).toMatch(/## Traces \([1-9]\d*\)/); + expect(traces).toMatch(/^\| [0-9a-f]{32} \|/m); }); - test("logs-analysis: runs an errors analysis", async ({ mcp }) => { + test("logs-analysis: reports a failed request", async ({ mcp }) => { + const missing = await callRaw(mcp, "localstack-aws-client", { + command: `s3api head-bucket --bucket ${PREFIX}-missing-bucket`, + }); + expect(missing.isError, missing.text).toBe(true); const logs = await call(mcp, "localstack-logs-analysis", { analysisType: "errors" }); - expect(logs).toMatch(/# 🚨 LocalStack Errors|βœ… No errors found/); + expect(logs).toContain("# 🚨 LocalStack Errors"); + expect(logs).toMatch(/HeadBucket/); }); - test("iam-policy-analyzer: switches to soft mode", async ({ mcp }) => { - expect(await call(mcp, "localstack-iam-policy-analyzer", { action: "get-status" })).toContain( - "**LocalStack IAM Enforcement Status**" - ); - await call(mcp, "localstack-iam-policy-analyzer", { action: "set-mode", mode: "SOFT_MODE" }); - expect(await call(mcp, "localstack-iam-policy-analyzer", { action: "get-status" })).toContain( - "SOFT_MODE" - ); + // ENFORCED, then SOFT_MODE: a set-mode that changes nothing fails one of the two read-backs. + test("iam-policy-analyzer: switches enforcement modes", async ({ mcp }) => { + const tool = "localstack-iam-policy-analyzer"; + try { + for (const mode of ["ENFORCED", "SOFT_MODE"]) { + await call(mcp, tool, { action: "set-mode", mode }); + expect(await call(mcp, tool, { action: "get-status" })).toContain( + `**Current Mode:** \`${mode}\`` + ); + } + } finally { + await callRaw(mcp, tool, { action: "set-mode", mode: "DISABLED" }); + } }); test("chaos-injector: injects, reads and clears latency", async ({ mcp }) => { const tool = "localstack-chaos-injector"; - expect(await call(mcp, tool, { action: "inject-latency", latency_ms: 100 })).toContain( - "Latency of 100ms has been injected" - ); - expect(await call(mcp, tool, { action: "get-latency" })).toContain( - "The current network latency is 100ms" - ); - expect(await call(mcp, tool, { action: "clear-latency" })).toContain( - "Network latency has been cleared" - ); + try { + expect(await call(mcp, tool, { action: "inject-latency", latency_ms: 100 })).toContain( + "Latency of 100ms has been injected" + ); + expect(await call(mcp, tool, { action: "get-latency" })).toContain( + "The current network latency is 100ms" + ); + expect(await call(mcp, tool, { action: "clear-latency" })).toContain( + "Network latency has been cleared" + ); + expect(await call(mcp, tool, { action: "get-latency" })).toContain( + "The current network latency is 0ms" + ); + } finally { + await callRaw(mcp, tool, { action: "clear-latency" }); + } }); + // The bucket is removed after the save, so only a load that restores the pod brings it back. test("cloud-pods: saves, loads and deletes a pod", async ({ mcp }) => { const tool = "localstack-cloud-pods"; + let saved = false; try { expect(await call(mcp, tool, { action: "save", pod_name: POD })).toContain( "was saved successfully" ); + saved = true; + await call(mcp, "localstack-aws-client", { command: `s3 rb s3://${BUCKET} --force` }); + expect(await call(mcp, "localstack-aws-client", { command: "s3 ls" })).not.toContain(BUCKET); expect(await call(mcp, tool, { action: "load", pod_name: POD })).toContain( "Your LocalStack instance has been restored" ); - } finally { + expect(await call(mcp, "localstack-aws-client", { command: "s3 ls" })).toContain(BUCKET); expect(await call(mcp, tool, { action: "delete", pod_name: POD })).toContain( "has been permanently deleted" ); + saved = false; + // A delete that did nothing would leave the pod, and a second delete would succeed. + const again = await callRaw(mcp, tool, { action: "delete", pod_name: POD }); + expect(again.isError, again.text).toBe(true); + } finally { + // Clean up without asserting, so a failed save or load reports its own error. + if (saved) await callRaw(mcp, tool, { action: "delete", pod_name: POD }); } }); From 209d1a0c86b4058aafb82d6415b796b046e2aed3 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Thu, 19 Nov 2026 15:17:56 +0100 Subject: [PATCH 08/13] ci: fail fork PRs that lack the LocalStack token instead of skipping the jobs A skipped job looks green, so a fork PR could merge without anyone noticing it was never tested against LocalStack. Both jobs now fail in their first step with an error that says how a maintainer can run them. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8417833..1824a3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,13 +36,19 @@ jobs: run: yarn build mcp-direct-tests: - # Fork PRs get no secrets, so they skip it; branches in this repo and Dependabot run it. - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-26.04 env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} steps: + # GitHub gives pull requests from forks no secrets. Fail loud rather than skip, so a + # fork PR never looks tested. + - name: Require the LocalStack auth token + if: env.LOCALSTACK_AUTH_TOKEN == '' + run: | + echo "::error::LOCALSTACK_AUTH_TOKEN is empty. GitHub gives pull requests from forks no secrets: before merging, a maintainer pushes this branch to the repository and opens a pull request from it." + exit 1 + - name: Checkout code uses: actions/checkout@v7 @@ -56,19 +62,23 @@ jobs: - name: Run MCP Server Tester run: yarn test:mcp:direct - env: - LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} # Each tool against a real LocalStack, without a model (tests/mcp/tools.spec.mjs). The first # test starts LocalStack through the management tool. mcp-tool-tests: - # Fork PRs get no secrets, so they skip it; branches in this repo and Dependabot run it. - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-26.04 env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} steps: + # GitHub gives pull requests from forks no secrets. Fail loud rather than skip, so a + # fork PR never looks tested. + - name: Require the LocalStack auth token + if: env.LOCALSTACK_AUTH_TOKEN == '' + run: | + echo "::error::LOCALSTACK_AUTH_TOKEN is empty. GitHub gives pull requests from forks no secrets: before merging, a maintainer pushes this branch to the repository and opens a pull request from it." + exit 1 + - name: Checkout code uses: actions/checkout@v7 From bae728ca02056c161ec9ea33eb6e5eaba4697904 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Thu, 19 Nov 2026 15:22:35 +0100 Subject: [PATCH 09/13] ci: fail the Docker integration test loud when the LocalStack token is missing Same as the CI jobs: on a fork PR, smoke (amd64) skipped the integration test and still showed green. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/docker.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e45a67e..f71bf02 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -37,6 +37,19 @@ jobs: runner: ubuntu-26.04-arm runs-on: ${{ matrix.runner }} steps: + # The integration test needs the token, and GitHub gives pull requests from forks no + # secrets. Fail loud rather than skip, so a fork PR never looks tested. + - name: Require the LocalStack auth token + # A step's if can't see its own env, so the shell checks the token. + if: matrix.arch == 'amd64' + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + run: | + if [ -z "$LOCALSTACK_AUTH_TOKEN" ]; then + echo "::error::LOCALSTACK_AUTH_TOKEN is empty. GitHub gives pull requests from forks no secrets: before merging, a maintainer pushes this branch to the repository and opens a pull request from it." + exit 1 + fi + - uses: actions/checkout@v7 - name: Set up Docker Buildx @@ -64,10 +77,7 @@ jobs: - name: Integration test Docker runtime # amd64 only: the deploys take minutes and do not depend on the image's architecture. - # Fork PRs get no secrets, so they skip it. - if: >- - matrix.arch == 'amd64' && - (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) + if: matrix.arch == 'amd64' env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} HARNESS_TOKEN_REAL: ${{ secrets.LOCALSTACK_AUTH_TOKEN != '' && '1' || '0' }} From 65d7879ceb960adb8c3926523c434a5982c72211 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Thu, 19 Nov 2026 15:22:53 +0100 Subject: [PATCH 10/13] TEMPORARY: run the evals workflow on push to the PR branch Co-Authored-By: Claude Opus 5.5 --- .github/workflows/mcp-evals.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/mcp-evals.yml b/.github/workflows/mcp-evals.yml index 4c2700e..2c833cf 100644 --- a/.github/workflows/mcp-evals.yml +++ b/.github/workflows/mcp-evals.yml @@ -10,6 +10,9 @@ on: default: "" schedule: - cron: "0 7 * * 1" + # TEMPORARY: run once on the PR branch, since workflow_dispatch needs the workflow on main. + push: + branches: [devx-1320-run-the-mcp-servers-llm-evals-in-ci-through] permissions: contents: read From de819ec226f9d8b3f2c5f2ca4a1cdb07fb857c96 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Thu, 19 Nov 2026 15:23:05 +0100 Subject: [PATCH 11/13] Revert the temporary push trigger for the evals workflow Co-Authored-By: Claude Opus 5.5 --- .github/workflows/mcp-evals.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/mcp-evals.yml b/.github/workflows/mcp-evals.yml index 2c833cf..4c2700e 100644 --- a/.github/workflows/mcp-evals.yml +++ b/.github/workflows/mcp-evals.yml @@ -10,9 +10,6 @@ on: default: "" schedule: - cron: "0 7 * * 1" - # TEMPORARY: run once on the PR branch, since workflow_dispatch needs the workflow on main. - push: - branches: [devx-1320-run-the-mcp-servers-llm-evals-in-ci-through] permissions: contents: read From 23f4c5c3853be48c37e2a31f831cb53be09af3fc Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Thu, 19 Nov 2026 15:43:31 +0100 Subject: [PATCH 12/13] ci: run the evals after every merge into main The weekly run stays, to catch changes outside this repo such as a new LocalStack image or model version. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/mcp-evals.yml | 6 +++++- README.md | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mcp-evals.yml b/.github/workflows/mcp-evals.yml index 4c2700e..40ed2e9 100644 --- a/.github/workflows/mcp-evals.yml +++ b/.github/workflows/mcp-evals.yml @@ -1,7 +1,10 @@ name: MCP evals -# The LLM-driven eval dataset through OpenRouter, weekly and on demand. +# The LLM-driven eval dataset through OpenRouter: after every merge into main, weekly (to catch +# changes outside this repo, such as a new LocalStack image or model version) and on demand. on: + push: + branches: [main] workflow_dispatch: inputs: model: @@ -15,6 +18,7 @@ permissions: contents: read # Runs share the Cloud Pod name mcptester-e2e-pod in one LocalStack account, so they must not overlap. +# Merges that land while a run is going wait; GitHub keeps only the newest waiting run. concurrency: group: mcp-evals cancel-in-progress: false diff --git a/README.md b/README.md index 9ed7384..6818a17 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ This repository includes [MCP Server Tester](https://github.com/gleanwork/mcp-se yarn test:mcp:evals ``` The evals check whether a model picks and combines the right tools for a goal; their prompts don't name the tools. They start, change and stop LocalStack on port 4566, so don't run them against an instance you're using. They need Docker, Terraform, [lstk](https://github.com/localstack/lstk) and the Snowflake CLI (`snow`) on the `PATH`, and the sample project's providers downloaded with `terraform -chdir=data/sample-terraform init`. - The [MCP evals workflow](./.github/workflows/mcp-evals.yml) runs them weekly and on demand; it needs the `OPENROUTER_API_KEY` repository secret. + The [MCP evals workflow](./.github/workflows/mcp-evals.yml) runs them after every merge into `main`, weekly and on demand; it needs the `OPENROUTER_API_KEY` repository secret. - Open the latest MCP Server Tester HTML report: ```bash npx mcp-server-tester open From d79163dff8ebff89086a096173e46aac9cf5c331 Mon Sep 17 00:00:00 2001 From: Joel Scheuner Date: Thu, 19 Nov 2026 15:50:07 +0100 Subject: [PATCH 13/13] ci: run the evals only for merges that can change them, and let forks build the image The eval trigger on main gets a path filter, so README edits or Dependabot action bumps don't start a run. The Docker token check moves right before the integration test, so a fork PR still builds the image and runs the startup smoke test before failing loud. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/docker.yml | 27 ++++++++++++++------------- .github/workflows/mcp-evals.yml | 21 +++++++++++++++++++-- README.md | 2 +- 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f71bf02..639a76f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -37,19 +37,6 @@ jobs: runner: ubuntu-26.04-arm runs-on: ${{ matrix.runner }} steps: - # The integration test needs the token, and GitHub gives pull requests from forks no - # secrets. Fail loud rather than skip, so a fork PR never looks tested. - - name: Require the LocalStack auth token - # A step's if can't see its own env, so the shell checks the token. - if: matrix.arch == 'amd64' - env: - LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} - run: | - if [ -z "$LOCALSTACK_AUTH_TOKEN" ]; then - echo "::error::LOCALSTACK_AUTH_TOKEN is empty. GitHub gives pull requests from forks no secrets: before merging, a maintainer pushes this branch to the repository and opens a pull request from it." - exit 1 - fi - - uses: actions/checkout@v7 - name: Set up Docker Buildx @@ -75,6 +62,20 @@ jobs: docker run -i --rm \ ${{ env.IMAGE_NAME }}:ci + # The integration test needs the token, and GitHub gives pull requests from forks no + # secrets. Fail loud rather than skip, so a fork PR never looks tested. Placed after the + # build and the startup smoke test, which need no secret and still run for forks. + - name: Require the LocalStack auth token + # A step's if can't see its own env, so the shell checks the token. + if: matrix.arch == 'amd64' + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + run: | + if [ -z "$LOCALSTACK_AUTH_TOKEN" ]; then + echo "::error::LOCALSTACK_AUTH_TOKEN is empty. GitHub gives pull requests from forks no secrets: before merging, a maintainer pushes this branch to the repository and opens a pull request from it." + exit 1 + fi + - name: Integration test Docker runtime # amd64 only: the deploys take minutes and do not depend on the image's architecture. if: matrix.arch == 'amd64' diff --git a/.github/workflows/mcp-evals.yml b/.github/workflows/mcp-evals.yml index 40ed2e9..086c679 100644 --- a/.github/workflows/mcp-evals.yml +++ b/.github/workflows/mcp-evals.yml @@ -1,10 +1,27 @@ name: MCP evals -# The LLM-driven eval dataset through OpenRouter: after every merge into main, weekly (to catch -# changes outside this repo, such as a new LocalStack image or model version) and on demand. +# The LLM-driven eval dataset through OpenRouter: after every merge into main that can change +# the result, weekly (to catch changes outside this repo, such as a new LocalStack image or model +# version) and on demand. on: push: branches: [main] + # What the MCP server, its tools and the evals are built from; README edits or Dependabot + # action bumps don't run the evals. + paths: + - "src/**" + - "data/**" + - "tests/mcp/**" + - "patches/**" + - "package.json" + - "yarn.lock" + - "playwright.config.mjs" + - "Dockerfile" + - "docker/**" + - "xmcp.config.*" + - "tsconfig*.json" + - ".github/workflows/mcp-evals.yml" + - ".github/actions/host-tools/**" workflow_dispatch: inputs: model: diff --git a/README.md b/README.md index 6818a17..f63a11a 100644 --- a/README.md +++ b/README.md @@ -275,7 +275,7 @@ This repository includes [MCP Server Tester](https://github.com/gleanwork/mcp-se yarn test:mcp:evals ``` The evals check whether a model picks and combines the right tools for a goal; their prompts don't name the tools. They start, change and stop LocalStack on port 4566, so don't run them against an instance you're using. They need Docker, Terraform, [lstk](https://github.com/localstack/lstk) and the Snowflake CLI (`snow`) on the `PATH`, and the sample project's providers downloaded with `terraform -chdir=data/sample-terraform init`. - The [MCP evals workflow](./.github/workflows/mcp-evals.yml) runs them after every merge into `main`, weekly and on demand; it needs the `OPENROUTER_API_KEY` repository secret. + The [MCP evals workflow](./.github/workflows/mcp-evals.yml) runs them after every merge into `main` that changes the server, its tools or the evals, weekly and on demand; it needs the `OPENROUTER_API_KEY` repository secret. - Open the latest MCP Server Tester HTML report: ```bash npx mcp-server-tester open