diff --git a/.github/actions/host-tools/action.yml b/.github/actions/host-tools/action.yml new file mode 100644 index 0000000..ad4244c --- /dev/null +++ b/.github/actions/host-tools/action.yml @@ -0,0 +1,33 @@ +name: Host tools +description: >- + Puts the Docker image's tools on the PATH for jobs that run the MCP server on the host: + always Terraform and uv (from the Dockerfile's host-tools stage) and lstk (docker/tools/npm), and + optionally the Snowflake CLI from docker/tools/snowflake. Versions change only in those files. +inputs: + snowflake: + description: Also install the Snowflake CLI. + required: false + default: "false" +runs: + using: composite + steps: + - name: Copy Terraform and uv out of the Dockerfile + shell: bash + run: | + docker build --target host-tools --output "type=local,dest=$RUNNER_TEMP/host-tools" . + echo "$RUNNER_TEMP/host-tools" >> "$GITHUB_PATH" + + - name: Install lstk + shell: bash + run: | + npm ci --prefix docker/tools/npm --omit=dev --ignore-scripts --no-audit --no-fund + echo "$GITHUB_WORKSPACE/docker/tools/npm/node_modules/.bin" >> "$GITHUB_PATH" + + - name: Install the Snowflake CLI + if: inputs.snowflake == 'true' + shell: bash + env: + UV_PROJECT_ENVIRONMENT: ${{ runner.temp }}/snowflake-cli + run: | + "$RUNNER_TEMP/host-tools/uv" sync --project docker/tools/snowflake --locked --no-install-project --no-build + echo "$UV_PROJECT_ENVIRONMENT/bin" >> "$GITHUB_PATH" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5a733f1..eab09f9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,6 +21,9 @@ updates: - dependency-name: zod update-types: - version-update:semver-minor + # patches/ holds a patch for this exact version, and patch-package fails the install in CI + # when it no longer applies. Bump it by hand and redo or drop the patch. + - dependency-name: "@gleanwork/mcp-server-tester" # CLIs inside the Docker image (aws-cdk, lstk) and the CDK sample app that CI deploys with them. - package-ecosystem: npm diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02ea70e..1824a3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,6 +41,14 @@ jobs: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} steps: + # GitHub gives pull requests from forks no secrets. Fail loud rather than skip, so a + # fork PR never looks tested. + - name: Require the LocalStack auth token + if: env.LOCALSTACK_AUTH_TOKEN == '' + run: | + echo "::error::LOCALSTACK_AUTH_TOKEN is empty. GitHub gives pull requests from forks no secrets: before merging, a maintainer pushes this branch to the repository and opens a pull request from it." + exit 1 + - name: Checkout code uses: actions/checkout@v7 @@ -54,5 +62,40 @@ jobs: - name: Run MCP Server Tester run: yarn test:mcp:direct - env: - LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + + # Each tool against a real LocalStack, without a model (tests/mcp/tools.spec.mjs). The first + # test starts LocalStack through the management tool. + mcp-tool-tests: + runs-on: ubuntu-26.04 + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + + steps: + # GitHub gives pull requests from forks no secrets. Fail loud rather than skip, so a + # fork PR never looks tested. + - name: Require the LocalStack auth token + if: env.LOCALSTACK_AUTH_TOKEN == '' + run: | + echo "::error::LOCALSTACK_AUTH_TOKEN is empty. GitHub gives pull requests from forks no secrets: before merging, a maintainer pushes this branch to the repository and opens a pull request from it." + exit 1 + + - name: Checkout code + uses: actions/checkout@v7 + + - name: Use Node.js 22 + uses: actions/setup-node@v7 + with: + node-version: 22.x + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: Install the image's tools + uses: ./.github/actions/host-tools + + # Pulled up front so the start stays within the management tool's 120s. + - name: Pull the LocalStack image + run: docker pull localstack/localstack-pro:latest + + - name: Run the tool tests + run: yarn test:mcp:tools diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e45a67e..639a76f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -62,12 +62,23 @@ jobs: docker run -i --rm \ ${{ env.IMAGE_NAME }}:ci + # The integration test needs the token, and GitHub gives pull requests from forks no + # secrets. Fail loud rather than skip, so a fork PR never looks tested. Placed after the + # build and the startup smoke test, which need no secret and still run for forks. + - name: Require the LocalStack auth token + # A step's if can't see its own env, so the shell checks the token. + if: matrix.arch == 'amd64' + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + run: | + if [ -z "$LOCALSTACK_AUTH_TOKEN" ]; then + echo "::error::LOCALSTACK_AUTH_TOKEN is empty. GitHub gives pull requests from forks no secrets: before merging, a maintainer pushes this branch to the repository and opens a pull request from it." + exit 1 + fi + - name: Integration test Docker runtime # amd64 only: the deploys take minutes and do not depend on the image's architecture. - # Fork PRs get no secrets, so they skip it. - if: >- - matrix.arch == 'amd64' && - (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) + if: matrix.arch == 'amd64' env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} HARNESS_TOKEN_REAL: ${{ secrets.LOCALSTACK_AUTH_TOKEN != '' && '1' || '0' }} diff --git a/.github/workflows/mcp-evals.yml b/.github/workflows/mcp-evals.yml new file mode 100644 index 0000000..086c679 --- /dev/null +++ b/.github/workflows/mcp-evals.yml @@ -0,0 +1,92 @@ +name: MCP evals + +# The LLM-driven eval dataset through OpenRouter: after every merge into main that can change +# the result, weekly (to catch changes outside this repo, such as a new LocalStack image or model +# version) and on demand. +on: + push: + branches: [main] + # What the MCP server, its tools and the evals are built from; README edits or Dependabot + # action bumps don't run the evals. + paths: + - "src/**" + - "data/**" + - "tests/mcp/**" + - "patches/**" + - "package.json" + - "yarn.lock" + - "playwright.config.mjs" + - "Dockerfile" + - "docker/**" + - "xmcp.config.*" + - "tsconfig*.json" + - ".github/workflows/mcp-evals.yml" + - ".github/actions/host-tools/**" + workflow_dispatch: + inputs: + model: + description: "OpenRouter model ID for every case (empty keeps the dataset's model)" + required: false + default: "" + schedule: + - cron: "0 7 * * 1" + +permissions: + contents: read + +# Runs share the Cloud Pod name mcptester-e2e-pod in one LocalStack account, so they must not overlap. +# Merges that land while a run is going wait; GitHub keeps only the newest waiting run. +concurrency: + group: mcp-evals + cancel-in-progress: false + +jobs: + mcp-evals: + runs-on: ubuntu-26.04 + # The spec itself times out after 90 minutes; the rest covers setup and the results upload. + timeout-minutes: 120 + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + EVAL_MODEL: ${{ inputs.model }} + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Use Node.js 22 + uses: actions/setup-node@v7 + with: + node-version: 22.x + + - name: Install dependencies + run: yarn install --frozen-lockfile + + # Terraform, lstk and the Snowflake CLI at the Docker image's versions. + - name: Install the image's tools + uses: ./.github/actions/host-tools + with: + snowflake: true + + # The eval host caps each tool call at the MCP SDK's 60s request timeout, and the + # deployer runs init, apply and output in one call. Download the AWS provider first. + - name: Download the sample project's Terraform providers + run: terraform -chdir=data/sample-terraform init -input=false + + # Pulled up front so a start inside a case doesn't time out on the download. + - name: Pull the LocalStack images + run: | + docker pull localstack/localstack-pro:latest + docker pull localstack/snowflake:latest + + - name: Run the eval dataset + run: yarn test:mcp:evals + + - name: Keep the eval results + if: always() + uses: actions/upload-artifact@v7 + with: + name: mcp-evals-results + path: .mcp-test-results + # The directory starts with a dot, which upload-artifact skips by default. + include-hidden-files: true + if-no-files-found: ignore diff --git a/Dockerfile b/Dockerfile index 9f297f5..ac5c2d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,7 @@ # aws-sam-cli docker/tools/sam/uv.lock (uv) # snowflake-cli docker/tools/snowflake/uv.lock (uv) # dockerode & co. yarn.lock (npm, same lock as CI) +# The host-tools stage hands the same terraform and uv binaries to CI jobs that run on the host. # Debian packages (python3, git, curl) come with the node image's bookworm base. # Both node FROM lines must stay identical, and their major must match `node-version` in the # workflows; src/core/dependency-pins.test.ts checks this. @@ -27,6 +28,12 @@ FROM hashicorp/terraform:1.17.2 AS terraform # uv installs the Python CLIs below; it is mounted at build time and not part of the image. FROM ghcr.io/astral-sh/uv:0.13.9 AS uv +# Terraform and uv for CI jobs that run on the host, at the image's versions +# (.github/actions/host-tools): docker build --target host-tools --output type=local,dest=DIR . +FROM scratch AS host-tools +COPY --from=terraform /bin/terraform / +COPY --from=uv /uv /uvx / + FROM node:22.23.5-bookworm-slim AS runtime ENV DEBIAN_FRONTEND=noninteractive \ PYTHONDONTWRITEBYTECODE=1 \ diff --git a/README.md b/README.md index 4ab0b40..f63a11a 100644 --- a/README.md +++ b/README.md @@ -262,17 +262,25 @@ This repository includes [MCP Server Tester](https://github.com/gleanwork/mcp-se ```bash yarn test:mcp:direct ``` -- Run Gemini-based MCP host evals: +- Run each tool against a running LocalStack, without a model (deterministic; starts LocalStack if it isn't running): ```bash - export GOOGLE_GENERATIVE_AI_API_KEY="" + export LOCALSTACK_AUTH_TOKEN="" + yarn test:mcp:tools + ``` + The tests deploy a sample project, switch IAM enforcement to soft mode and save a Cloud Pod, so point them at an instance you don't need: set `LOCALSTACK_PORT` and `MAIN_CONTAINER_NAME` to use one other than the default on 4566. They need Docker, Terraform and [lstk](https://github.com/localstack/lstk) on the `PATH`. +- Run MCP host evals through [OpenRouter](https://openrouter.ai) (default model `google/gemini-2.5-flash`; set `EVAL_MODEL` to any OpenRouter model ID to override it): + ```bash + export OPENROUTER_API_KEY="" export LOCALSTACK_AUTH_TOKEN="" yarn test:mcp:evals ``` + The evals check whether a model picks and combines the right tools for a goal; their prompts don't name the tools. They start, change and stop LocalStack on port 4566, so don't run them against an instance you're using. They need Docker, Terraform, [lstk](https://github.com/localstack/lstk) and the Snowflake CLI (`snow`) on the `PATH`, and the sample project's providers downloaded with `terraform -chdir=data/sample-terraform init`. + The [MCP evals workflow](./.github/workflows/mcp-evals.yml) runs them after every merge into `main` that changes the server, its tools or the evals, weekly and on demand; it needs the `OPENROUTER_API_KEY` repository secret. - Open the latest MCP Server Tester HTML report: ```bash npx mcp-server-tester open ``` -- Run both: +- Run all of them: ```bash yarn test:mcp ``` diff --git a/data/evals/comprehensive.json b/data/evals/comprehensive.json new file mode 100644 index 0000000..5177669 --- /dev/null +++ b/data/evals/comprehensive.json @@ -0,0 +1,266 @@ +{ + "name": "localstack-mcp-comprehensive", + "description": "Whether a model picks and combines the LocalStack MCP tools; tests/mcp/tools.spec.mjs checks each tool without a model. Prompts describe the goal, not the tool. One iteration per case. Each matchesPattern starts with a prefix that keeps it inside one tool call output: the tester matches against the whole simulation result as JSON, which also holds the scenario and the model answer.", + "cases": [ + { + "id": "scenario-management-status", + "description": "localstack-management: status, and a start when LocalStack isn't running. Runs first, so later cases have a running LocalStack.", + "mode": "mcp_host", + "scenario": "Is my local AWS emulator running? If it isn't, bring it up. Tell me once it is ready to accept requests.", + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, + "expect": { + "toolsTriggered": { + "calls": [ + { + "name": "localstack-management", + "required": true + } + ], + "order": "any" + }, + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*Services initialized: [1-9]\\d*/\\d+"] + } + }, + { + "id": "scenario-cloud-pods-roundtrip", + "description": "localstack-cloud-pods: save, load and delete a named pod. Runs second, right after LocalStack starts: the eval host times out each tool call at ~60s (MCP SDK default; the tester's requestTimeoutMs is not applied), and a save takes longer the more state later cases add.", + "mode": "mcp_host", + "scenario": "Save the current state of my local AWS emulator as a remote snapshot named mcptester-e2e-pod that a teammate could restore. If a snapshot with that name already exists, delete it and save again. Then restore mcptester-e2e-pod into my emulator to check that it works, and finally delete the snapshot so nothing is left behind. If a save, restore or delete times out, retry it once before giving up.", + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0, + "maxToolCalls": 15 + }, + "expect": { + "toolsTriggered": { + "calls": [ + { + "name": "localstack-cloud-pods", + "arguments": { "action": "save", "pod_name": "mcptester-e2e-pod" }, + "required": true + }, + { + "name": "localstack-cloud-pods", + "arguments": { "action": "load", "pod_name": "mcptester-e2e-pod" }, + "required": true + }, + { + "name": "localstack-cloud-pods", + "arguments": { "action": "delete", "pod_name": "mcptester-e2e-pod" }, + "required": true + } + ], + "order": "strict" + }, + "matchesPattern": [ + "\"output\":\"(?:[^\"\\\\]|\\\\.)*was saved successfully", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*was loaded\\. Your LocalStack instance has been restored", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*has been permanently deleted" + ] + } + }, + { + "id": "scenario-deploy-terraform-project", + "description": "localstack-deployer: deploy the sample Terraform project, which creates the bucket the next case lists.", + "mode": "mcp_host", + "scenario": "Deploy the Terraform project in this repository's data/sample-terraform directory to my local AWS emulator, then tell me its outputs.", + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, + "expect": { + "toolsTriggered": { + "calls": [ + { + "name": "localstack-deployer", + "arguments": { "action": "deploy" }, + "required": true + } + ], + "order": "any" + }, + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*Apply complete!"] + } + }, + { + "id": "scenario-aws-client-list-buckets", + "description": "localstack-aws-client: list S3 buckets, which include the one the deploy case created.", + "mode": "mcp_host", + "scenario": "Which S3 buckets exist in my local AWS emulator right now?", + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, + "expect": { + "toolsTriggered": { + "calls": [ + { + "name": "localstack-aws-client", + "required": true + } + ], + "order": "any" + }, + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*mcpjam-eval-bucket"] + } + }, + { + "id": "scenario-iam-soft-mode", + "description": "localstack-iam-policy-analyzer: switch IAM enforcement to soft mode from a symptom, not the mode's name.", + "mode": "mcp_host", + "scenario": "My Lambda function keeps failing with AccessDenied errors in my local AWS emulator. Instead of blocking those calls, let them through and only log which permissions are missing. Then check which mode is active.", + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, + "expect": { + "toolsTriggered": { + "calls": [ + { + "name": "localstack-iam-policy-analyzer", + "arguments": { "action": "set-mode", "mode": "SOFT_MODE" }, + "required": true + } + ], + "order": "any" + }, + "matchesPattern": [ + "\"output\":\"(?:[^\"\\\\]|\\\\.)*IAM enforcement mode has been set to `SOFT_MODE`", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*\\*\\*Current Mode:\\*\\* `SOFT_MODE`" + ] + } + }, + { + "id": "scenario-chaos-latency-cycle", + "description": "localstack-chaos-injector: inject, read back and clear network latency.", + "mode": "mcp_host", + "scenario": "Simulate a slow network: make every request to my local AWS emulator 100 milliseconds slower, check that the delay is in effect, then remove it again and check that it is gone.", + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, + "expect": { + "toolsTriggered": { + "calls": [ + { + "name": "localstack-chaos-injector", + "arguments": { "action": "inject-latency", "latency_ms": 100 }, + "required": true + }, + { + "name": "localstack-chaos-injector", + "arguments": { "action": "get-latency" }, + "required": true + }, + { + "name": "localstack-chaos-injector", + "arguments": { "action": "clear-latency" }, + "required": true + } + ], + "order": "strict" + }, + "matchesPattern": [ + "\"output\":\"(?:[^\"\\\\]|\\\\.)*Latency of 100ms has been injected", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*The current network latency is 100ms", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*Network latency has been cleared", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*The current network latency is 0ms" + ] + } + }, + { + "id": "scenario-app-inspector-traces", + "description": "localstack-app-inspector: list traces, enabling App Inspector first if needed.", + "mode": "mcp_host", + "scenario": "Show me the most recent traces of the AWS API calls made against my local emulator. If tracing is turned off, turn it on first.", + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, + "expect": { + "toolsTriggered": { + "calls": [ + { + "name": "localstack-app-inspector", + "arguments": { "action": "list-traces" }, + "required": true + } + ], + "order": "any" + }, + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*## Traces"] + } + }, + { + "id": "scenario-docs-search", + "description": "localstack-docs: a how-it-works question the docs answer, next to IAM tools that act on the running instance.", + "mode": "mcp_host", + "scenario": "How does IAM enforcement work in LocalStack, and how do its enforcement modes differ? Look it up in the documentation instead of answering from memory.", + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0 + }, + "expect": { + "toolsTriggered": { + "calls": [ + { + "name": "localstack-docs", + "required": true + } + ], + "order": "any" + }, + "matchesPattern": ["\"output\":\"(?:[^\"\\\\]|\\\\.)*\\*\\*[1-9]\\d* results? found\\*\\*"] + } + }, + { + "id": "scenario-snowflake-full-flow", + "description": "localstack-snowflake-client: start emulator, check connection, run SQL. Runs LAST: the Snowflake stack and the AWS stack cannot run at the same time (both own the localstack-main container), so the AWS emulator must be stopped first and this case leaves the Snowflake emulator running.", + "mode": "mcp_host", + "scenario": "Work with the LocalStack Snowflake emulator: 1) Using the LocalStack management tool with service set to snowflake, start the Snowflake emulator. If it reports that it is already running, continue to step 2. If starting fails because a LocalStack container is already running, that container is the AWS emulator: stop it with the management tool, then start the Snowflake emulator with service snowflake again. If a start attempt times out, keep checking the Snowflake status with the management tool (service snowflake) until it reports ready — do not give up on a timeout. 2) Use the LocalStack Snowflake client to check the connection to the emulator. 3) Use the LocalStack Snowflake client to execute the SQL file data/sample-sql/snowflake_test.sql. 4) Run the query SHOW TABLES IN demo_db.public and tell me whether the customers table exists.", + "mcpHostConfig": { + "provider": "openrouter", + "model": "google/gemini-2.5-flash", + "temperature": 0, + "maxToolCalls": 15 + }, + "expect": { + "toolsTriggered": { + "calls": [ + { + "name": "localstack-management", + "arguments": { "service": "snowflake" }, + "required": true + }, + { + "name": "localstack-snowflake-client", + "arguments": { "action": "check-connection" }, + "required": true + }, + { + "name": "localstack-snowflake-client", + "arguments": { "action": "execute" }, + "required": true + } + ], + "order": "strict" + }, + "matchesPattern": [ + "\"output\":\"(?:[^\"\\\\]|\\\\.)*(?:\\| Status\\s+\\| OK\\s+\\|)", + "\"output\":\"(?:[^\"\\\\]|\\\\.)*(?:\\| CUSTOMERS\\s+\\|)" + ] + } + } + ] +} diff --git a/data/evals/gemini-comprehensive.json b/data/evals/gemini-comprehensive.json deleted file mode 100644 index a03bc9c..0000000 --- a/data/evals/gemini-comprehensive.json +++ /dev/null @@ -1,345 +0,0 @@ -{ - "name": "localstack-mcp-gemini-comprehensive", - "description": "Scenario-based Gemini evals covering LocalStack MCP tools.", - "cases": [ - { - "id": "scenario-management-status", - "description": "localstack-management: status check of the AWS emulator.", - "mode": "mcp_host", - "scenario": "Check the current status of the LocalStack AWS emulator using the LocalStack management tool. If it is not running, start it. Report whether it is ready to accept requests.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-management", - "arguments": { "action": "status" }, - "required": true - } - ], - "order": "any" - } - } - }, - { - "id": "scenario-deploy-terraform-project", - "description": "localstack-deployer: deploy the sample Terraform project.", - "mode": "mcp_host", - "scenario": "Deploy the Terraform project located in the data/sample-terraform directory of this repository to LocalStack using the LocalStack deployer tool, then report the Terraform outputs.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { "name": "localstack-deployer", "arguments": { "action": "deploy" }, "required": true } - ], - "order": "any" - } - } - }, - { - "id": "scenario-aws-client-list-buckets", - "description": "localstack-aws-client: run a pinned AWS CLI command.", - "mode": "mcp_host", - "scenario": "Using the LocalStack AWS client tool, run the AWS CLI command \"s3 ls\" against LocalStack. Pass the command string exactly as s3 ls, with no aws or awslocal prefix and no extra flags. Then tell me which S3 buckets exist.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-aws-client", - "arguments": { "command": "s3 ls" }, - "required": true - } - ], - "order": "any" - } - } - }, - { - "id": "scenario-logs-analysis-errors", - "description": "localstack-logs-analysis: errors analysis of container logs.", - "mode": "mcp_host", - "scenario": "Use the LocalStack logs analysis tool to run an 'errors' analysis of the recent LocalStack container logs and report which AWS API calls failed.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-logs-analysis", - "arguments": { "analysisType": "errors" }, - "required": true - } - ], - "order": "any" - } - } - }, - { - "id": "scenario-app-inspector-traces", - "description": "localstack-app-inspector: status check plus trace listing.", - "mode": "mcp_host", - "scenario": "Using the LocalStack App Inspector tool, first check whether App Inspector is currently enabled, enable it only if it is disabled, and then list the most recent App Inspector traces.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-app-inspector", - "arguments": { "action": "get-status" }, - "required": true - }, - { - "name": "localstack-app-inspector", - "arguments": { "action": "list-traces" }, - "required": true - } - ], - "order": "any" - } - } - }, - { - "id": "scenario-iam-soft-mode", - "description": "localstack-iam-policy-analyzer: read mode, then set SOFT_MODE.", - "mode": "mcp_host", - "scenario": "Check the current IAM enforcement mode in LocalStack using the IAM policy analyzer, then set the enforcement mode to SOFT_MODE with the set-mode action. Perform the set-mode call even if the current mode is already SOFT_MODE, so the setting is explicitly confirmed.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-iam-policy-analyzer", - "arguments": { "action": "get-status" }, - "required": true - }, - { - "name": "localstack-iam-policy-analyzer", - "arguments": { "action": "set-mode", "mode": "SOFT_MODE" }, - "required": true - } - ], - "order": "any" - } - } - }, - { - "id": "scenario-state-inspect", - "description": "localstack-state-management: read-only inspect of runtime state.", - "mode": "mcp_host", - "scenario": "Using the LocalStack state management tool, inspect the in-memory state of the running LocalStack instance and tell me which AWS services currently hold state. Do not export, import, or reset anything.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-state-management", - "arguments": { "action": "inspect" }, - "required": true - } - ], - "order": "any" - } - } - }, - { - "id": "scenario-cloud-pods-roundtrip", - "description": "localstack-cloud-pods: save, load, and delete a named pod. NOTE: the eval host times out individual tool calls at ~60s (MCP SDK default; the tester's requestTimeoutMs is not applied), so this case needs a reasonably fresh container whose state saves quickly.", - "mode": "mcp_host", - "scenario": "Using the LocalStack Cloud Pods tool, save the current LocalStack state to a remote Cloud Pod named mcptester-e2e-pod. If saving fails because a pod with that name already exists, delete the existing pod and save again. If a save, load, or delete attempt times out, retry that same operation once before giving up. After saving, load the Cloud Pod mcptester-e2e-pod back into LocalStack. Finally, delete the Cloud Pod mcptester-e2e-pod so nothing is left behind.", - "mcpHostConfig": { - "provider": "google", - "model": "gemini-2.5-flash", - "temperature": 0, - "maxToolCalls": 15 - }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-cloud-pods", - "arguments": { "action": "save", "pod_name": "mcptester-e2e-pod" }, - "required": true - }, - { - "name": "localstack-cloud-pods", - "arguments": { "action": "load", "pod_name": "mcptester-e2e-pod" }, - "required": true - }, - { - "name": "localstack-cloud-pods", - "arguments": { "action": "delete", "pod_name": "mcptester-e2e-pod" }, - "required": true - } - ], - "order": "strict" - } - } - }, - { - "id": "scenario-chaos-latency-cycle", - "description": "localstack-chaos-injector: inject, verify, and clear latency.", - "mode": "mcp_host", - "scenario": "Use the LocalStack chaos injector to add 100 milliseconds of network latency to LocalStack, then read back the currently configured latency to verify it was applied, and finally clear the latency so LocalStack returns to normal.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-chaos-injector", - "arguments": { "action": "inject-latency", "latency_ms": 100 }, - "required": true - }, - { - "name": "localstack-chaos-injector", - "arguments": { "action": "get-latency" }, - "required": true - }, - { - "name": "localstack-chaos-injector", - "arguments": { "action": "clear-latency" }, - "required": true - } - ], - "order": "strict" - } - } - }, - { - "id": "scenario-extensions-marketplace", - "description": "localstack-extensions: browse the marketplace catalog.", - "mode": "mcp_host", - "scenario": "Browse the LocalStack Extensions marketplace and show me which extensions are available to install. I want the marketplace catalog of available extensions, not the list of extensions that are already installed.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-extensions", - "arguments": { "action": "available" }, - "required": true - } - ], - "order": "any" - } - } - }, - { - "id": "scenario-replicator-read-only", - "description": "localstack-aws-replicator: read-only supported-resources and job listing.", - "mode": "mcp_host", - "scenario": "Using the LocalStack AWS replicator tool, first show me which resource types the AWS replicator supports for replication into LocalStack, and then list any replication jobs that currently exist. Do not start any replication job.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-aws-replicator", - "arguments": { "action": "list-resources" }, - "required": true - }, - { - "name": "localstack-aws-replicator", - "arguments": { "action": "list" }, - "required": true - } - ], - "order": "any" - } - } - }, - { - "id": "scenario-ephemeral-list", - "description": "localstack-ephemeral-instances: read-only listing.", - "mode": "mcp_host", - "scenario": "List all LocalStack Ephemeral Instances that currently exist in my LocalStack Cloud account using the ephemeral instances tool, and summarize their names and statuses. Do not create, modify, or delete any instance.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-ephemeral-instances", - "arguments": { "action": "list" }, - "required": true - } - ], - "order": "any" - } - } - }, - { - "id": "scenario-docs-search", - "description": "localstack-docs: documentation search.", - "mode": "mcp_host", - "scenario": "Search the official LocalStack documentation for how IAM policy enforcement modes work in LocalStack and summarize what the docs say. Use the LocalStack documentation search tool, not the IAM tools that act on the running instance.", - "mcpHostConfig": { "provider": "google", "model": "gemini-2.5-flash", "temperature": 0 }, - "iterations": 5, - "accuracyThreshold": 0.8, - "expect": { - "toolsTriggered": { - "calls": [{ "name": "localstack-docs", "required": true }], - "order": "any" - } - } - }, - { - "id": "scenario-snowflake-full-flow", - "description": "localstack-snowflake-client: start emulator, check connection, run SQL. Runs LAST: the Snowflake stack and the AWS stack cannot run at the same time (both own the localstack-main container), so the AWS emulator must be stopped first and this case leaves the Snowflake emulator running.", - "mode": "mcp_host", - "scenario": "Work with the LocalStack Snowflake emulator: 1) Using the LocalStack management tool with service set to snowflake, start the Snowflake emulator. If it reports that it is already running, continue to step 2. If starting fails because a LocalStack container is already running, that container is the AWS emulator: stop it with the management tool, then start the Snowflake emulator with service snowflake again. If a start attempt times out, keep checking the Snowflake status with the management tool (service snowflake) until it reports ready — do not give up on a timeout. 2) Use the LocalStack Snowflake client to check the connection to the emulator. 3) Use the LocalStack Snowflake client to execute the SQL file data/sample-sql/snowflake_test.sql. 4) Run the query SHOW TABLES IN demo_db.public and tell me whether the customers table exists.", - "mcpHostConfig": { - "provider": "google", - "model": "gemini-2.5-flash", - "temperature": 0, - "maxToolCalls": 15 - }, - "iterations": 3, - "accuracyThreshold": 0.66, - "expect": { - "toolsTriggered": { - "calls": [ - { - "name": "localstack-management", - "arguments": { "service": "snowflake" }, - "required": true - }, - { - "name": "localstack-snowflake-client", - "arguments": { "action": "check-connection" }, - "required": true - }, - { - "name": "localstack-snowflake-client", - "arguments": { "action": "execute" }, - "required": true - } - ], - "order": "strict" - } - } - } - ] -} diff --git a/package.json b/package.json index 8c3b7ad..b8775dd 100644 --- a/package.json +++ b/package.json @@ -12,11 +12,13 @@ "dev": "xmcp dev", "start": "node dist/cli.js", "prepack": "yarn build", + "prepare": "patch-package", "format": "prettier --write .", "test": "jest", "test:mcp:direct": "yarn build && playwright test -c playwright.config.mjs tests/mcp/direct.spec.mjs tests/mcp/azure-offline.spec.mjs", - "test:mcp:evals": "yarn build && playwright test -c playwright.config.mjs tests/mcp/evals-gemini.spec.mjs", - "test:mcp": "yarn test:mcp:direct && yarn test:mcp:evals" + "test:mcp:evals": "yarn build && playwright test -c playwright.config.mjs tests/mcp/evals.spec.mjs", + "test:mcp:tools": "yarn build && playwright test -c playwright.config.mjs tests/mcp/tools.spec.mjs", + "test:mcp": "yarn test:mcp:direct && yarn test:mcp:tools && yarn test:mcp:evals" }, "dependencies": { "ajv": "^8.18.0", @@ -26,6 +28,7 @@ "devDependencies": { "@clack/prompts": "^1.8.1", "@gleanwork/mcp-server-tester": "1.1.1", + "@openrouter/ai-sdk-provider": "^2.0.0", "@playwright/test": "^1.64.1", "@types/dockerode": "^4.0.2", "@types/jest": "^30.0.0", @@ -35,6 +38,7 @@ "eslint-config-prettier": "^10.1.8", "jest": "^30.5.2", "jsonc-parser": "^3.3.1", + "patch-package": "^8.0.1", "posthog-node": "5.73.1", "prettier": "^3.9.10", "swc-loader": "^0.2.6", diff --git a/patches/@gleanwork+mcp-server-tester+1.1.1.patch b/patches/@gleanwork+mcp-server-tester+1.1.1.patch new file mode 100644 index 0000000..03da55b --- /dev/null +++ b/patches/@gleanwork+mcp-server-tester+1.1.1.patch @@ -0,0 +1,36 @@ +diff --git a/node_modules/@gleanwork/mcp-server-tester/dist/index.cjs b/node_modules/@gleanwork/mcp-server-tester/dist/index.cjs +index 1361291..4c0d0e2 100644 +--- a/node_modules/@gleanwork/mcp-server-tester/dist/index.cjs ++++ b/node_modules/@gleanwork/mcp-server-tester/dist/index.cjs +@@ -6965,9 +6965,10 @@ function createVercelOrchestrator() { + const totalDurationMs = Date.now() - llmStart; + const llmDurationMs = totalDurationMs - mcpDurationMs; + const hostUsage = result.usage ? { +- inputTokens: result.usage.promptTokens ?? 0, +- outputTokens: result.usage.completionTokens ?? 0, +- totalCostUsd: 0, ++ // AI SDK 5+ names: totalUsage covers every step; OpenRouter reports each step's cost. ++ inputTokens: (result.totalUsage ?? result.usage).inputTokens ?? 0, ++ outputTokens: (result.totalUsage ?? result.usage).outputTokens ?? 0, ++ totalCostUsd: (result.steps ?? []).reduce((sum, step) => sum + (step.providerMetadata?.openrouter?.usage?.cost ?? 0), 0), + durationMs: llmDurationMs + } : void 0; + const conversationHistory = (result.steps ?? []).flatMap((step) => { +diff --git a/node_modules/@gleanwork/mcp-server-tester/dist/index.js b/node_modules/@gleanwork/mcp-server-tester/dist/index.js +index 71c1ef0..8c444e9 100644 +--- a/node_modules/@gleanwork/mcp-server-tester/dist/index.js ++++ b/node_modules/@gleanwork/mcp-server-tester/dist/index.js +@@ -6938,9 +6938,10 @@ function createVercelOrchestrator() { + const totalDurationMs = Date.now() - llmStart; + const llmDurationMs = totalDurationMs - mcpDurationMs; + const hostUsage = result.usage ? { +- inputTokens: result.usage.promptTokens ?? 0, +- outputTokens: result.usage.completionTokens ?? 0, +- totalCostUsd: 0, ++ // AI SDK 5+ names: totalUsage covers every step; OpenRouter reports each step's cost. ++ inputTokens: (result.totalUsage ?? result.usage).inputTokens ?? 0, ++ outputTokens: (result.totalUsage ?? result.usage).outputTokens ?? 0, ++ totalCostUsd: (result.steps ?? []).reduce((sum, step) => sum + (step.providerMetadata?.openrouter?.usage?.cost ?? 0), 0), + durationMs: llmDurationMs + } : void 0; + const conversationHistory = (result.steps ?? []).flatMap((step) => { diff --git a/src/core/dependency-pins.test.ts b/src/core/dependency-pins.test.ts index 84b0d47..b279c97 100644 --- a/src/core/dependency-pins.test.ts +++ b/src/core/dependency-pins.test.ts @@ -78,7 +78,11 @@ describe("dependency pins", () => { // Runners are either a literal label or `${{ matrix.runner }}`, whose values are checked // as literals; any other expression could hide an unpinned label. const docker = read(".github/workflows/docker.yml"); - const workflow = [docker, read(".github/workflows/ci.yml")].join("\n"); + const workflow = [ + docker, + read(".github/workflows/ci.yml"), + read(".github/workflows/mcp-evals.yml"), + ].join("\n"); for (const [, value] of workflow.matchAll(/^[ \t]*runs-on:[ \t]*(.*)$/gm)) { if (value.startsWith("${{")) expect(value).toBe("${{ matrix.runner }}"); } @@ -93,6 +97,37 @@ describe("dependency pins", () => { expect(docker).toMatch(/^[ \t]*(?:-[ \t]*)?runner:[ \t]*ubuntu-[\d.]+-arm[ \t]*$/m); }); + it("gives CI jobs the image's Terraform and uv, from the Dockerfile's host-tools stage", () => { + // Dependabot bumps the FROM lines, not action inputs or install scripts, so any other way + // of installing these tools in CI would drift from the image. + const stage = dockerfileInstructions().join("\n"); + expect(stage).toMatch(/^FROM scratch AS host-tools$/m); + expect(stage).toMatch(/^COPY --from=terraform \S+ \/$/m); + expect(stage).toMatch(/^COPY --from=uv \S+ \S+ \/$/m); + + const files = [ + ...fs.readdirSync(path.join(ROOT, ".github/workflows")).map((f) => `.github/workflows/${f}`), + ".github/actions/host-tools/action.yml", + ]; + const otherInstall = + /setup-terraform|setup-uv|terraform_version|\b(?:curl|wget)\b.*\b(?:hashicorp\.com|astral\.sh|astral-sh)\b|\b(?:pipx?|brew|apt(?:-get)?|mise|asdf) install\b.*\b(?:terraform|uv)\b/g; + for (const file of files) { + expect(`${file}: ${read(file).match(otherInstall) ?? []}`).toBe(`${file}: `); + } + + // Jobs that run the tool tests or the evals need the tools; without the action they would + // use whatever the runner image ships. Jobs are split at their two-space-indented keys. + for (const file of fs.readdirSync(path.join(ROOT, ".github/workflows"))) { + const jobs = read(`.github/workflows/${file}`).split(/^jobs:\s*$/m)[1] ?? ""; + for (const job of jobs.split(/^(?= [\w-]+:\s*$)/m)) { + if (!/\b(?:yarn|npm|pnpm)(?: run)? test:mcp(?::tools|:evals)?\b(?!:)/.test(job)) continue; + expect( + `${file} ${job.trim().split(":")[0]}: ${/uses: \.\/\.github\/actions\/host-tools\b/.test(job)}` + ).toBe(`${file} ${job.trim().split(":")[0]}: true`); + } + } + }); + it("installs CI dependencies from the same frozen lockfile as the image", () => { for (const file of fs.readdirSync(path.join(ROOT, ".github/workflows"))) { for (const match of read(`.github/workflows/${file}`).matchAll(/run:\s*(yarn\b.*)/g)) { diff --git a/tests/mcp/evals-gemini.spec.mjs b/tests/mcp/evals-gemini.spec.mjs deleted file mode 100644 index 528c337..0000000 --- a/tests/mcp/evals-gemini.spec.mjs +++ /dev/null @@ -1,36 +0,0 @@ -import { loadEvalDataset, runEvalDataset } from "@gleanwork/mcp-server-tester"; -import { test, expect } from "@gleanwork/mcp-server-tester/fixtures/mcp"; - -function requireEnv(name) { - const value = process.env[name]; - if (!value || !value.trim()) { - throw new Error(`Missing required environment variable: ${name}`); - } - return value; -} - -test.describe("Gemini comprehensive eval", () => { - test.describe.configure({ timeout: 5400000 }); - - test("single comprehensive eval dataset passes", async ({ mcp }, testInfo) => { - requireEnv("GOOGLE_GENERATIVE_AI_API_KEY"); - requireEnv("LOCALSTACK_AUTH_TOKEN"); - - const dataset = await loadEvalDataset("./data/evals/gemini-comprehensive.json"); - const result = await runEvalDataset({ dataset }, { mcp, testInfo }); - const caseResults = result.caseResults || []; - const passed = caseResults.filter((entry) => entry?.pass === true).length; - const failed = caseResults.filter((entry) => entry?.pass !== true); - - if (failed.length > 0) { - for (const entry of failed) { - console.error(`Eval case failed: ${entry.id}`); - console.error(JSON.stringify(entry, null, 2)); - } - } - console.log(`Eval pass rate: ${passed}/${caseResults.length}`); - - const passRate = caseResults.length > 0 ? passed / caseResults.length : 1; - expect(passRate).toBeGreaterThanOrEqual(0.75); - }); -}); diff --git a/tests/mcp/evals.spec.mjs b/tests/mcp/evals.spec.mjs new file mode 100644 index 0000000..05ed49e --- /dev/null +++ b/tests/mcp/evals.spec.mjs @@ -0,0 +1,78 @@ +import { loadEvalDataset, runEvalDataset } from "@gleanwork/mcp-server-tester"; +import { test, expect } from "@gleanwork/mcp-server-tester/fixtures/mcp"; + +function requireEnv(name) { + const value = process.env[name]; + if (!value || !value.trim()) { + throw new Error(`Missing required environment variable: ${name}`); + } + return value; +} + +test.describe("OpenRouter comprehensive eval", () => { + test.describe.configure({ timeout: 5400000 }); + + test("single comprehensive eval dataset passes", async ({ mcp }, testInfo) => { + requireEnv("OPENROUTER_API_KEY"); + requireEnv("LOCALSTACK_AUTH_TOKEN"); + + const dataset = await loadEvalDataset("./data/evals/comprehensive.json"); + // EVAL_MODEL swaps the OpenRouter model for every case, e.g. anthropic/claude-sonnet-4.5. + const model = process.env.EVAL_MODEL?.trim(); + if (model) { + for (const evalCase of dataset.cases) { + if (evalCase.mcpHostConfig) evalCase.mcpHostConfig.model = model; + } + } + const result = await runEvalDataset({ dataset }, { mcp, testInfo }); + const caseResults = result.caseResults || []; + // The tester leaves iterations with infrastructure errors (rate limits, 5xx, network) out of + // the pass rate, so 1 good iteration out of 5 can pass a case. Count them as failures instead. + const thresholds = new Map( + dataset.cases.map((c) => [c.id, Math.ceil((c.accuracyThreshold ?? 1) * (c.iterations ?? 1))]) + ); + // The tester drops a tool result's isError and keeps only its text, which this server starts + // with ❌ on an error. A model can still read a usable answer out of an error's text, so each + // required call needs at least one matching call that didn't fail. + const cases = new Map(dataset.cases.map((c) => [c.id, c])); + const matches = (call, expected) => + call.name === expected.name && + Object.entries(expected.arguments ?? {}).every( + ([key, value]) => JSON.stringify(call.arguments?.[key]) === JSON.stringify(value) + ); + const failedCalls = (entry, iteration) => + (cases.get(entry.id)?.expect?.toolsTriggered?.calls ?? []) + .filter((expected) => expected.required) + .filter( + (expected) => + !(iteration.response?.toolCalls ?? []).some( + (call) => matches(call, expected) && !String(call.output ?? "").startsWith("❌") + ) + ) + .map((expected) => expected.name); + const passesIteration = (entry, iteration) => + iteration.pass === true && failedCalls(entry, iteration).length === 0; + const passesCase = (entry) => + entry?.pass === true && + (entry.iterationResults ?? [entry]).filter((r) => passesIteration(entry, r)).length >= + thresholds.get(entry.id); + const passed = caseResults.filter(passesCase).length; + const failed = caseResults.filter((entry) => !passesCase(entry)); + + if (failed.length > 0) { + for (const entry of failed) { + const errored = (entry.iterationResults ?? [entry]).map((r) => failedCalls(entry, r)); + console.error( + `Eval case failed: ${entry.id} (infrastructure errors: ${entry.infrastructureErrorCount ?? 0}; required calls that only returned errors: ${JSON.stringify(errored)})` + ); + console.error(JSON.stringify(entry, null, 2)); + } + } + console.log(`Eval pass rate: ${passed}/${caseResults.length}`); + + // Every case must pass. A case's own accuracyThreshold already absorbs flaky iterations, so a + // case that fails here failed most of them, e.g. because its tool is broken. + expect(caseResults).toHaveLength(dataset.cases.length); + expect(failed.map((entry) => entry.id)).toEqual([]); + }); +}); diff --git a/tests/mcp/tools.spec.mjs b/tests/mcp/tools.spec.mjs new file mode 100644 index 0000000..f1bf52a --- /dev/null +++ b/tests/mcp/tools.spec.mjs @@ -0,0 +1,189 @@ +import { expect, test } from "@gleanwork/mcp-server-tester/fixtures/mcp"; + +// One deterministic call sequence per tool against a running LocalStack: no model involved, so +// these run on every pull request. The evals (evals.spec.mjs) cover whether a model picks and +// combines the tools; these cover whether each tool works. +// +// Each test checks the state a tool leaves behind, read back through another call, not only the +// tool's own success message: a tool that skips its API call but still reports success must fail. +// +// The tests share one LocalStack and run in order: the first starts LocalStack if needed, the +// deploy creates the bucket and queue that later tests look for. To use an instance other than +// the default on 4566, set LOCALSTACK_PORT and MAIN_CONTAINER_NAME. + +// The tester's callTool applies the MCP SDK's 60s request timeout; a start, a deploy or a Cloud +// Pod save can take longer, so call the SDK client directly. +const CALL_TIMEOUT_MS = 300_000; +// A name_prefix other than data/sample-terraform's default, so the test sees the variable applied. +const PREFIX = "mcp-tools"; +const BUCKET = `${PREFIX}-bucket`; +const QUEUE = `${PREFIX}-queue`; +// Unique per run and attempt, so a re-run never meets a pod an aborted attempt left behind. +const POD = `mcp-tools-test-${process.env.GITHUB_RUN_ID ?? Date.now()}-${process.env.GITHUB_RUN_ATTEMPT ?? 1}`; + +async function callRaw(mcp, name, args) { + const result = await mcp.client.callTool({ name, arguments: args }, undefined, { + timeout: CALL_TIMEOUT_MS, + }); + const text = (result.content ?? []).map((block) => block.text ?? "").join("\n"); + return { isError: result.isError === true, text }; +} + +// Returns the tool's text, failing with that text when the tool reports an error. +async function call(mcp, name, args) { + const { isError, text } = await callRaw(mcp, name, args); + expect(isError, `${name} ${JSON.stringify(args)} failed:\n${text}`).toBe(false); + return text; +} + +test.describe("tools against a running LocalStack", () => { + test.describe.configure({ mode: "serial", timeout: 600_000 }); + + test.beforeAll(() => { + if (!process.env.LOCALSTACK_AUTH_TOKEN?.trim()) { + throw new Error("Missing required environment variable: LOCALSTACK_AUTH_TOKEN"); + } + }); + + test("management: starts LocalStack and reports it ready", async ({ mcp }) => { + const before = await callRaw(mcp, "localstack-management", { action: "status" }); + if (!/Services initialized/.test(before.text)) { + await call(mcp, "localstack-management", { action: "start" }); + } + const status = await call(mcp, "localstack-management", { action: "status" }); + expect(status).toMatch(/Services initialized: [1-9]\d*\/\d+/); + }); + + // Disabling first means the enable has to change something, whatever an earlier run left. + test("app-inspector: disables and enables tracing", async ({ mcp }) => { + const tool = "localstack-app-inspector"; + await call(mcp, tool, { action: "set-status", status: "disabled" }); + expect(await call(mcp, tool, { action: "get-status" })).toMatch(/\*\*Status:\*\* disabled\b/i); + await call(mcp, tool, { action: "set-status", status: "enabled" }); + expect(await call(mcp, tool, { action: "get-status" })).toMatch(/\*\*Status:\*\* enabled\b/i); + }); + + test("deployer: applies the sample Terraform project with a variable", async ({ mcp }) => { + const report = await call(mcp, "localstack-deployer", { + action: "deploy", + projectType: "terraform", + directory: "data/sample-terraform", + variables: { name_prefix: PREFIX }, + }); + expect(report).toContain("Apply complete!"); + // The outputs table, not the apply log, which also prints the outputs. + expect(report).toContain(`| **bucket_name** | \`${BUCKET}\` |`); + }); + + test("aws-client: lists the deployed bucket and queue", async ({ mcp }) => { + const tool = "localstack-aws-client"; + expect(await call(mcp, tool, { command: "s3 ls" })).toContain(BUCKET); + // SQS is regional, so this also catches a client that talks to the wrong region. + expect(await call(mcp, tool, { command: "sqs list-queues" })).toContain(QUEUE); + }); + + test("state-management: shows S3 state", async ({ mcp }) => { + const state = await call(mcp, "localstack-state-management", { action: "inspect" }); + expect(state).toContain("## LocalStack State Inspect"); + expect(state).toMatch(/\bs3\b/i); + }); + + test("app-inspector: lists traces of the deploy", async ({ mcp }) => { + const traces = await call(mcp, "localstack-app-inspector", { action: "list-traces" }); + expect(traces).toMatch(/## Traces \([1-9]\d*\)/); + expect(traces).toMatch(/^\| [0-9a-f]{32} \|/m); + }); + + test("logs-analysis: reports a failed request", async ({ mcp }) => { + const missing = await callRaw(mcp, "localstack-aws-client", { + command: `s3api head-bucket --bucket ${PREFIX}-missing-bucket`, + }); + expect(missing.isError, missing.text).toBe(true); + const logs = await call(mcp, "localstack-logs-analysis", { analysisType: "errors" }); + expect(logs).toContain("# 🚨 LocalStack Errors"); + expect(logs).toMatch(/HeadBucket/); + }); + + // ENFORCED, then SOFT_MODE: a set-mode that changes nothing fails one of the two read-backs. + test("iam-policy-analyzer: switches enforcement modes", async ({ mcp }) => { + const tool = "localstack-iam-policy-analyzer"; + try { + for (const mode of ["ENFORCED", "SOFT_MODE"]) { + await call(mcp, tool, { action: "set-mode", mode }); + expect(await call(mcp, tool, { action: "get-status" })).toContain( + `**Current Mode:** \`${mode}\`` + ); + } + } finally { + await callRaw(mcp, tool, { action: "set-mode", mode: "DISABLED" }); + } + }); + + test("chaos-injector: injects, reads and clears latency", async ({ mcp }) => { + const tool = "localstack-chaos-injector"; + try { + expect(await call(mcp, tool, { action: "inject-latency", latency_ms: 100 })).toContain( + "Latency of 100ms has been injected" + ); + expect(await call(mcp, tool, { action: "get-latency" })).toContain( + "The current network latency is 100ms" + ); + expect(await call(mcp, tool, { action: "clear-latency" })).toContain( + "Network latency has been cleared" + ); + expect(await call(mcp, tool, { action: "get-latency" })).toContain( + "The current network latency is 0ms" + ); + } finally { + await callRaw(mcp, tool, { action: "clear-latency" }); + } + }); + + // The bucket is removed after the save, so only a load that restores the pod brings it back. + test("cloud-pods: saves, loads and deletes a pod", async ({ mcp }) => { + const tool = "localstack-cloud-pods"; + let saved = false; + try { + expect(await call(mcp, tool, { action: "save", pod_name: POD })).toContain( + "was saved successfully" + ); + saved = true; + await call(mcp, "localstack-aws-client", { command: `s3 rb s3://${BUCKET} --force` }); + expect(await call(mcp, "localstack-aws-client", { command: "s3 ls" })).not.toContain(BUCKET); + expect(await call(mcp, tool, { action: "load", pod_name: POD })).toContain( + "Your LocalStack instance has been restored" + ); + expect(await call(mcp, "localstack-aws-client", { command: "s3 ls" })).toContain(BUCKET); + expect(await call(mcp, tool, { action: "delete", pod_name: POD })).toContain( + "has been permanently deleted" + ); + saved = false; + // A delete that did nothing would leave the pod, and a second delete would succeed. + const again = await callRaw(mcp, tool, { action: "delete", pod_name: POD }); + expect(again.isError, again.text).toBe(true); + } finally { + // Clean up without asserting, so a failed save or load reports its own error. + if (saved) await callRaw(mcp, tool, { action: "delete", pod_name: POD }); + } + }); + + test("aws-replicator: lists resource types and jobs", async ({ mcp }) => { + const tool = "localstack-aws-replicator"; + expect(await call(mcp, tool, { action: "list-resources" })).toContain( + "## AWS Replicator Supported Resources" + ); + expect(await call(mcp, tool, { action: "list" })).toContain("## AWS Replicator Jobs"); + }); + + test("extensions: lists the marketplace", async ({ mcp }) => { + expect(await call(mcp, "localstack-extensions", { action: "available" })).toMatch( + /[1-9]\d* extensions available\. Install any/ + ); + }); + + test("ephemeral-instances: lists instances", async ({ mcp }) => { + expect(await call(mcp, "localstack-ephemeral-instances", { action: "list" })).toContain( + "## Ephemeral Instances" + ); + }); +}); diff --git a/yarn.lock b/yarn.lock index 443a457..e240cef 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1890,6 +1890,11 @@ ts-checker-rspack-plugin "^1.8.0" typescript "^5.9.3" +"@yarnpkg/lockfile@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz#e77a97fbd345b76d83245edcd17d393b1b41fb31" + integrity sha512-GpSwvyXOcOOlV70vbnzjj4fW5xW/FdUF6nQEt1ENy7m4ZCczi1+/buVUPAqmGfqznsORNFzUMjctTIp8a9tuCQ== + abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" @@ -2184,6 +2189,13 @@ brace-expansion@^5.0.8: dependencies: balanced-match "^4.0.2" +braces@^3.0.3: + version "3.0.3" + resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" + integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== + dependencies: + fill-range "^7.1.1" + browserslist@^4.24.0: version "4.28.7" resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.28.7.tgz#409046517fccd2e51cdc20f077454b7141184028" @@ -2247,7 +2259,17 @@ call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2: es-errors "^1.3.0" function-bind "^1.1.2" -call-bound@^1.0.2: +call-bind@^1.0.8: + version "1.0.9" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.9.tgz#39a644700c80bc7d0ca9102fc6d1d43b2fd7eee7" + integrity sha512-a/hy+pNsFUTR+Iz8TCJvXudKVLAnz/DyeSUo10I5yvFDQJBFU2s9uqQpoSrJlroHUKoKqzg+epxyP9lqFdzfBQ== + dependencies: + call-bind-apply-helpers "^1.0.2" + es-define-property "^1.0.1" + get-intrinsic "^1.3.0" + set-function-length "^1.2.2" + +call-bound@^1.0.2, call-bound@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/call-bound/-/call-bound-1.0.4.tgz#238de935d2a2a692928c538c7ccfa91067fd062a" integrity sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg== @@ -2305,6 +2327,11 @@ chownr@^1.1.1: resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== +ci-info@^3.7.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.9.0.tgz#4279a62028a7b1f262f3473fc9605f5e218c59b4" + integrity sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ== + ci-info@^4.2.0: version "4.4.0" resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-4.4.0.tgz#7d54eff9f54b45b62401c26032696eb59c8bd18c" @@ -2500,6 +2527,15 @@ default-browser@^5.2.1: bundle-name "^4.1.0" default-browser-id "^5.0.0" +define-data-property@^1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/define-data-property/-/define-data-property-1.1.4.tgz#894dc141bb7d3060ae4366f6a0107e68fbe48c5e" + integrity sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A== + dependencies: + es-define-property "^1.0.0" + es-errors "^1.3.0" + gopd "^1.0.1" + define-lazy-prop@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz#dbb19adfb746d7fc6d734a06b72f4a00d021255f" @@ -2642,7 +2678,7 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -es-define-property@^1.0.1: +es-define-property@^1.0.0, es-define-property@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/es-define-property/-/es-define-property-1.0.1.tgz#983eb2f9a6724e9303f61addf011c72e09e0b0fa" integrity sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g== @@ -2948,6 +2984,13 @@ figures@^6.1.0: dependencies: is-unicode-supported "^2.0.0" +fill-range@^7.1.1: + version "7.1.1" + resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" + integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== + dependencies: + to-regex-range "^5.0.1" + finalhandler@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/finalhandler/-/finalhandler-2.1.1.tgz#a2c517a6559852bcdb06d1f8bd7f51b68fad8099" @@ -2968,6 +3011,13 @@ find-up@^4.0.0, find-up@^4.1.0: locate-path "^5.0.0" path-exists "^4.0.0" +find-yarn-workspace-root@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/find-yarn-workspace-root/-/find-yarn-workspace-root-2.0.0.tgz#f47fb8d239c900eb78179aa81b66673eac88f7bd" + integrity sha512-1IMnbjt4KzsQfnhnzNd8wUEgXZ44IzZaZmnLYx7D5FZlaHt2gW20Cri8Q+E/t5tIj4+epTBub+2Zxu/vNILzqQ== + dependencies: + micromatch "^4.0.2" + foreground-child@^3.1.0: version "3.3.1" resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.3.1.tgz#32e8e9ed1b68a3497befb9ac2b6adf92a638576f" @@ -3015,6 +3065,15 @@ fs-constants@^1.0.0: resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== +fs-extra@^10.0.0: + version "10.1.0" + resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.1.0.tgz#02873cfbc4084dde127eaa5f9905eef2325d1abf" + integrity sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ== + dependencies: + graceful-fs "^4.2.0" + jsonfile "^6.0.1" + universalify "^2.0.0" + function-bind@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" @@ -3073,7 +3132,7 @@ get-east-asian-width@^1.0.0, get-east-asian-width@^1.3.1: resolved "https://registry.yarnpkg.com/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz#216900f91df11a8b2c198c3e1d93d6c035a776b9" integrity sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA== -get-intrinsic@^1.2.5, get-intrinsic@^1.2.6, get-intrinsic@^1.3.0: +get-intrinsic@^1.2.4, get-intrinsic@^1.2.5, get-intrinsic@^1.2.6, get-intrinsic@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.3.0.tgz#743f0e3b6964a93a5491ed1bffaae054d7f98d01" integrity sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ== @@ -3172,11 +3231,16 @@ google-logging-utils@^1.0.0: resolved "https://registry.yarnpkg.com/google-logging-utils/-/google-logging-utils-1.2.0.tgz#89dd643f986c765e61bfa2d01cada2292fe5bfd7" integrity sha512-WE9av4wKDZgRjBwgVUabocx8T6/7o3Ca1Fat46FXDhXVAFibzNadedcOXrdgd1Kzmk8tsk/9ZH89Wyf/SqeZ3A== -gopd@^1.2.0: +gopd@^1.0.1, gopd@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/gopd/-/gopd-1.2.0.tgz#89f56b8217bdbc8802bd299df6d7f1081d7e51a1" integrity sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg== +graceful-fs@^4.1.11, graceful-fs@^4.1.6, graceful-fs@^4.2.0: + version "4.2.12" + resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.12.tgz#7f6162dee07bc600281cea1b038e5f7cf215a6c2" + integrity sha512-iuyGF2E4oafQfPsRrG2wtY2ILupMhvtdrhzrxB+0VcjsPpSPttpfYdjnGFGsTi5MaJFjI1jOKHPQ6UTbTj9TAw== + graceful-fs@^4.2.11: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" @@ -3207,6 +3271,13 @@ has-flag@^4.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== +has-property-descriptors@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz#963ed7d071dc7bf5f084c5bfbe0d1b6222586854" + integrity sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg== + dependencies: + es-define-property "^1.0.0" + has-symbols@^1.0.3, has-symbols@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/has-symbols/-/has-symbols-1.1.0.tgz#fc9c6a783a084951d0b971fe1018de813707a338" @@ -3382,6 +3453,11 @@ is-arrayish@^0.2.1: resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" integrity sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg== +is-docker@^2.0.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-2.2.1.tgz#33eeabe23cfe86f14bde4408a02c0cfb853acdaa" + integrity sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ== + is-docker@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/is-docker/-/is-docker-3.0.0.tgz#90093aa3106277d8a77a5910dbae71747e15a200" @@ -3433,6 +3509,11 @@ is-inside-container@^1.0.0: dependencies: is-docker "^3.0.0" +is-number@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" + integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== + is-promise@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-4.0.0.tgz#42ff9f84206c1991d26debf520dd5c01042dd2f3" @@ -3453,6 +3534,13 @@ is-unsafe@^2.0.3: resolved "https://registry.yarnpkg.com/is-unsafe/-/is-unsafe-2.0.3.tgz#c8bc72c9d354dc9483dab9c26ff69bf5ec68d7b0" integrity sha512-ZBllczCCO267vgdFb6Tq3pv+LhHaCiGMdr+ZsuL+OopruAg7S+3ZDs1vD0RPC0uSrkjzbraxaGjtX1YBxjPeFQ== +is-wsl@^2.1.1: + version "2.2.0" + resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-2.2.0.tgz#74a4c76e77ca9fd3f932f290c17ea326cd157271" + integrity sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww== + dependencies: + is-docker "^2.0.0" + is-wsl@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/is-wsl/-/is-wsl-3.1.1.tgz#327897b26832a3eb117da6c27492d04ca132594f" @@ -3460,6 +3548,11 @@ is-wsl@^3.1.0: dependencies: is-inside-container "^1.0.0" +isarray@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723" + integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw== + isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" @@ -4005,6 +4098,17 @@ json-schema@^0.4.0: resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.4.0.tgz#f7de4cf6efab838ebaeb3236474cbba5a1930ab5" integrity sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA== +json-stable-stringify@^1.0.2: + version "1.3.0" + resolved "https://registry.yarnpkg.com/json-stable-stringify/-/json-stable-stringify-1.3.0.tgz#8903cfac42ea1a0f97f35d63a4ce0518f0cc6a70" + integrity sha512-qtYiSSFlwot9XHtF9bD9c7rwKjr+RecWT//ZnPvSmEjpV5mmPOCN4j8UjY5hbjNkOwZ/jQv3J6R1/pL7RwgMsg== + dependencies: + call-bind "^1.0.8" + call-bound "^1.0.4" + isarray "^2.0.5" + jsonify "^0.0.1" + object-keys "^1.1.1" + json5@^2.2.3: version "2.2.3" resolved "https://registry.yarnpkg.com/json5/-/json5-2.2.3.tgz#78cd6f1a19bdc12b73db5ad0c61efd66c1e29283" @@ -4015,6 +4119,20 @@ jsonc-parser@^3.3.1: resolved "https://registry.yarnpkg.com/jsonc-parser/-/jsonc-parser-3.3.1.tgz#f2a524b4f7fd11e3d791e559977ad60b98b798b4" integrity sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ== +jsonfile@^6.0.1: + version "6.2.1" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.2.1.tgz#b6e31717f22cc37330b081ce0051ed5de53af2f6" + integrity sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q== + dependencies: + universalify "^2.0.0" + optionalDependencies: + graceful-fs "^4.1.6" + +jsonify@^0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/jsonify/-/jsonify-0.0.1.tgz#2aa3111dae3d34a0f151c63f3a45d995d9420978" + integrity sha512-2/Ki0GcmuqSrgFyelQq9M05y7PS0mEwuIzrf3f1fPqkVDVRvZrPZtVSMHxdgo8Aq0sxAOb/cr2aqqA3LeWHVPg== + jsonparse@^1.2.0: version "1.3.1" resolved "https://registry.yarnpkg.com/jsonparse/-/jsonparse-1.3.1.tgz#3f4dae4a91fac315f71062f8521cc239f1366280" @@ -4045,6 +4163,13 @@ jws@^4.0.0: jwa "^2.0.1" safe-buffer "^5.0.1" +klaw-sync@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/klaw-sync/-/klaw-sync-6.0.0.tgz#1fd2cfd56ebb6250181114f0a581167099c2b28c" + integrity sha512-nIeuVSzdCCs6TDPTqI8w1Yre34sSq7AkZ4B3sfOBbI2CgVSB4Du4aLQijFU2+lhAFCwt9+42Hel6lQNIv6AntQ== + dependencies: + graceful-fs "^4.1.11" + leven@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/leven/-/leven-3.1.0.tgz#77891de834064cccba82ae7842bb6b14a13ed7f2" @@ -4158,6 +4283,14 @@ merge-stream@^2.0.0: resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60" integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== +micromatch@^4.0.2: + version "4.0.8" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" + integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== + dependencies: + braces "^3.0.3" + picomatch "^2.3.1" + mime-db@1.52.0: version "1.52.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" @@ -4206,7 +4339,7 @@ minimatch@^9.0.4: dependencies: brace-expansion "^2.0.2" -minimist@^1.2.8: +minimist@^1.2.6, minimist@^1.2.8: version "1.2.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== @@ -4314,6 +4447,11 @@ object-inspect@^1.13.3, object-inspect@^1.13.4: resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.13.4.tgz#8375265e21bc20d0fa582c22e1b13485d6e00213" integrity sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew== +object-keys@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e" + integrity sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA== + on-finished@^2.4.1: version "2.4.1" resolved "https://registry.yarnpkg.com/on-finished/-/on-finished-2.4.1.tgz#58c8c44116e54845ad57f14ab10b03533184ac3f" @@ -4345,6 +4483,14 @@ open@^10.1.0: is-inside-container "^1.0.0" wsl-utils "^0.1.0" +open@^7.4.2: + version "7.4.2" + resolved "https://registry.yarnpkg.com/open/-/open-7.4.2.tgz#b8147e26dcf3e426316c730089fd71edd29c2321" + integrity sha512-MVHddDVweXZF3awtlAS+6pgKLlm/JgxZ90+/NBurBoQctVOOB/zDdVjcyPzQ+0laDGbsWgrRkflI65sQeOgT9Q== + dependencies: + is-docker "^2.0.0" + is-wsl "^2.1.1" + p-limit@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1" @@ -4403,6 +4549,26 @@ patch-console@^2.0.0: resolved "https://registry.yarnpkg.com/patch-console/-/patch-console-2.0.0.tgz#9023f4665840e66f40e9ce774f904a63167433bb" integrity sha512-0YNdUceMdaQwoKce1gatDScmMo5pu/tfABfnzEqeG0gtTmd7mh/WcwgUjtAeOU7N8nFFlbQBnFK2gXW5fGvmMA== +patch-package@^8.0.1: + version "8.0.1" + resolved "https://registry.yarnpkg.com/patch-package/-/patch-package-8.0.1.tgz#79d02f953f711e06d1f8949c8a13e5d3d7ba1a60" + integrity sha512-VsKRIA8f5uqHQ7NGhwIna6Bx6D9s/1iXlA1hthBVBEbkq+t4kXD0HHt+rJhf/Z+Ci0F/HCB2hvn0qLdLG+Qxlw== + dependencies: + "@yarnpkg/lockfile" "^1.1.0" + chalk "^4.1.2" + ci-info "^3.7.0" + cross-spawn "^7.0.3" + find-yarn-workspace-root "^2.0.0" + fs-extra "^10.0.0" + json-stable-stringify "^1.0.2" + klaw-sync "^6.0.0" + minimist "^1.2.6" + open "^7.4.2" + semver "^7.5.3" + slash "^2.0.0" + tmp "^0.2.4" + yaml "^2.2.2" + path-exists@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" @@ -4451,7 +4617,7 @@ picocolors@^1.1.1: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== -picomatch@^2.0.4: +picomatch@^2.0.4, picomatch@^2.3.1: version "2.3.2" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.2.tgz#5a942915e26b372dc0f0e6753149a16e6b1c5601" integrity sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA== @@ -4760,6 +4926,18 @@ serve-static@^2.2.0: parseurl "^1.3.3" send "^1.2.0" +set-function-length@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/set-function-length/-/set-function-length-1.2.2.tgz#aac72314198eaed975cf77b2c3b6b880695e5449" + integrity sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg== + dependencies: + define-data-property "^1.1.4" + es-errors "^1.3.0" + function-bind "^1.1.2" + get-intrinsic "^1.2.4" + gopd "^1.0.1" + has-property-descriptors "^1.0.2" + setprototypeof@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" @@ -4832,6 +5010,11 @@ sisteransi@^2.0.0: resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-2.0.0.tgz#d3d7a694075c2b17094cf1a691afd7937643b936" integrity sha512-2WCqLdt12eN6ehLI50S2/ZxoowXU3DD1KVPRyVP0WEsFahrGZZmCdq1K59qjJljMRIDTwx1D9YaRK+hDneBUMg== +slash@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/slash/-/slash-2.0.0.tgz#de552851a1759df3a8f206535442f5ec4ddeab44" + integrity sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A== + slash@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/slash/-/slash-3.0.0.tgz#6539be870c165adbd5240220dbe361f1bc4d4634" @@ -5103,6 +5286,18 @@ through@2, through@^2.3.8, through@~2.3, through@~2.3.4: resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== +tmp@^0.2.4: + version "0.2.7" + resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.7.tgz#26f4db11d1601ce8012dcb8a798ece1c06a99059" + integrity sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw== + +to-regex-range@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" + integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ== + dependencies: + is-number "^7.0.0" + toidentifier@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/toidentifier/-/toidentifier-1.0.1.tgz#3be34321a88a820ed1bd80dfaa33e479fbb8dd35" @@ -5212,6 +5407,11 @@ undici@^7.24.0: resolved "https://registry.yarnpkg.com/undici/-/undici-7.30.1.tgz#360b3de104b94abc52729aa1a4a07e027aa71f73" integrity sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg== +universalify@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-2.0.1.tgz#168efc2180964e6386d061e094df61afe239b18d" + integrity sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw== + unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" @@ -5397,6 +5597,11 @@ yallist@^3.0.2: resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== +yaml@^2.2.2: + version "2.9.1" + resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.9.1.tgz#c16233fb31944e705cfefaff38795587f57588ce" + integrity sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw== + yargs-parser@^21.1.1: version "21.1.1" resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35"